Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
+1
-1
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
|
||||
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs."""
|
||||
|
||||
# The version.py should be independent library, and we always import the
|
||||
# version library first. Such assumption is critical for some customization.
|
||||
|
||||
+21
-22
@@ -1835,6 +1835,7 @@ class rocm_aiter_ops:
|
||||
- MLA decode: mla_decode_fwd
|
||||
- Quantization: per_tensor_quant, per_token_quant, group_fp8_quant
|
||||
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
|
||||
|
||||
"""
|
||||
|
||||
_MOE_DISPATCH_POLICY: int | None = None
|
||||
@@ -1874,8 +1875,7 @@ class rocm_aiter_ops:
|
||||
|
||||
@classmethod
|
||||
def refresh_env_variables(cls):
|
||||
"""
|
||||
Since the environment variables are assigned when the module is imported,
|
||||
"""Since the environment variables are assigned when the module is imported,
|
||||
This is a helper function to reload all the env variables from
|
||||
the environment variables.
|
||||
for example, after monkey patching the env variables in the unit test,
|
||||
@@ -1900,8 +1900,7 @@ class rocm_aiter_ops:
|
||||
|
||||
@staticmethod
|
||||
def get_aiter_activation_type(activation_str: str) -> "ActivationType | None":
|
||||
"""
|
||||
Given an activation type as a string, returns the corresponding aiter ActivationType enum.
|
||||
"""Given an activation type as a string, returns the corresponding aiter ActivationType enum.
|
||||
Supported activation types: "no", "none", "silu", "gelu", "swiglu".
|
||||
Returns None if the mapping fails.
|
||||
|
||||
@@ -1910,6 +1909,7 @@ class rocm_aiter_ops:
|
||||
|
||||
Returns:
|
||||
Aiter ActivationType enum value, or None if not found.
|
||||
|
||||
"""
|
||||
# Import only locally, since aiter may not always be available.
|
||||
try:
|
||||
@@ -1933,8 +1933,7 @@ class rocm_aiter_ops:
|
||||
|
||||
@staticmethod
|
||||
def get_aiter_quant_type(quant_type_str: str) -> "QuantType | None":
|
||||
"""
|
||||
Given a quantization type as a string, returns the corresponding aiter QuantType enum.
|
||||
"""Given a quantization type as a string, returns the corresponding aiter QuantType enum.
|
||||
Supported quantization types: "no", "per_tensor", "per_token", "per_1x32", "per_1x128", "per_128x128".
|
||||
Returns None if the mapping fails.
|
||||
|
||||
@@ -1943,6 +1942,7 @@ class rocm_aiter_ops:
|
||||
|
||||
Returns:
|
||||
Aiter QuantType enum value, or None if not found.
|
||||
|
||||
"""
|
||||
try:
|
||||
from aiter import QuantType
|
||||
@@ -1974,7 +1974,8 @@ class rocm_aiter_ops:
|
||||
enabled aiter. Only aiter's Triton kernels exist on gfx12 (no CK build),
|
||||
so this deliberately stays off the gfx9/CK `@if_aiter_supported` umbrella
|
||||
and gates only the Triton paths rdna4 uses. The gfx12 analog of
|
||||
`is_enabled()`."""
|
||||
`is_enabled()`.
|
||||
"""
|
||||
if not current_platform.is_rocm() or not IS_AITER_FOUND:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_rdna4
|
||||
@@ -3357,8 +3358,7 @@ class rocm_aiter_ops:
|
||||
nLane: int,
|
||||
gate_up: bool,
|
||||
) -> "torch.Tensor":
|
||||
"""
|
||||
Shuffles the weight tensor into (A16W4) layout for AITER kernels.
|
||||
"""Shuffles the weight tensor into (A16W4) layout for AITER kernels.
|
||||
|
||||
Args:
|
||||
tensor: The input weight tensor to be shuffled.
|
||||
@@ -3367,6 +3367,7 @@ class rocm_aiter_ops:
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The shuffled tensor.
|
||||
|
||||
"""
|
||||
from aiter.ops.shuffle import shuffle_weight_a16w4
|
||||
|
||||
@@ -3378,8 +3379,7 @@ class rocm_aiter_ops:
|
||||
num_experts: int,
|
||||
gate_up: bool,
|
||||
) -> "torch.Tensor":
|
||||
"""
|
||||
Shuffles the scale tensor into (A16W4) layout for AITER kernels.
|
||||
"""Shuffles the scale tensor into (A16W4) layout for AITER kernels.
|
||||
|
||||
Args:
|
||||
tensor: The input scale tensor to be shuffled.
|
||||
@@ -3388,6 +3388,7 @@ class rocm_aiter_ops:
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The shuffled scale tensor.
|
||||
|
||||
"""
|
||||
from aiter.ops.shuffle import shuffle_scale_a16w4
|
||||
|
||||
@@ -3404,8 +3405,7 @@ class rocm_aiter_ops:
|
||||
def shuffle_weights(
|
||||
*tensors: torch.Tensor, layout: tuple[int, int] = (16, 16)
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""
|
||||
Applies shuffle_weight function from AITER to each
|
||||
"""Applies shuffle_weight function from AITER to each
|
||||
input tensor and returns them.
|
||||
|
||||
Rearranges (shuffles) the input tensor/s
|
||||
@@ -3418,6 +3418,7 @@ class rocm_aiter_ops:
|
||||
|
||||
Returns:
|
||||
A Tuple of shuffled tensors.
|
||||
|
||||
"""
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
@@ -3467,8 +3468,7 @@ class rocm_aiter_ops:
|
||||
out: torch.Tensor | None = None,
|
||||
sink_ptr: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Flash attention with variable length sequences.
|
||||
"""Flash attention with variable length sequences.
|
||||
|
||||
This function is NOT wrapped with @is_aiter_supported decorator
|
||||
to allow explicit backend selection via attention_config to work
|
||||
@@ -3537,8 +3537,7 @@ class rocm_aiter_ops:
|
||||
V_QScale: torch.Tensor,
|
||||
out_: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Paged attention forward pass using assembly kernel.
|
||||
"""Paged attention forward pass using assembly kernel.
|
||||
|
||||
This function is NOT wrapped with @is_aiter_supported decorator
|
||||
to allow explicit backend selection via attention_config to work
|
||||
@@ -3580,8 +3579,7 @@ class rocm_aiter_ops:
|
||||
out_: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
):
|
||||
"""
|
||||
Paged attention common function.
|
||||
"""Paged attention common function.
|
||||
|
||||
This function is NOT wrapped with @is_aiter_supported decorator
|
||||
to allow explicit backend selection via attention_config to work
|
||||
@@ -3625,8 +3623,7 @@ class rocm_aiter_ops:
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
@@ -3645,6 +3642,7 @@ class rocm_aiter_ops:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
from aiter.ops.mhc import mhc_pre
|
||||
|
||||
@@ -3735,7 +3733,7 @@ class rocm_aiter_ops:
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre using the pre-mix carried from the previous sublayer.
|
||||
"""MHC pre using the pre-mix carried from the previous sublayer.
|
||||
|
||||
Same gates as :meth:`mhc_pre`, but the stream collapse applies the
|
||||
caller's ``pre_mix`` instead of the one computed here, and the one
|
||||
@@ -3762,6 +3760,7 @@ class rocm_aiter_ops:
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
|
||||
"""
|
||||
from aiter.ops.mhc import (
|
||||
get_mhc_fused_post_pre_config,
|
||||
|
||||
+57
-53
@@ -39,8 +39,7 @@ def create_fp4_scale_tensor(
|
||||
device: torch.device,
|
||||
is_sf_swizzled_layout: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Allocate the output scale tensor for scaled_fp4_quant.
|
||||
"""Allocate the output scale tensor for scaled_fp4_quant.
|
||||
|
||||
When is_sf_swizzled_layout=True, we use rounded values to store the
|
||||
swizzled scales. Due to the requirement of the Tensor Core, the minimum
|
||||
@@ -73,8 +72,7 @@ def create_fp4_output_tensors(
|
||||
is_sf_swizzled_layout: bool,
|
||||
padded_n: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Allocate both output tensors for scaled_fp4_quant:
|
||||
"""Allocate both output tensors for scaled_fp4_quant:
|
||||
(quantized_output, output_scale).
|
||||
|
||||
Must match the C++ scaled_fp4_quant_func allocation exactly when
|
||||
@@ -265,9 +263,18 @@ def vocab_parallel_embedding(
|
||||
Args:
|
||||
input_ids: ``[num_tokens]`` int32 or int64 token ids.
|
||||
weight: ``[num_embeddings_per_partition, embedding_dim]`` shard.
|
||||
org_vocab_start_index: First original vocab id owned by this rank.
|
||||
org_vocab_end_index: One past the last original vocab id owned by
|
||||
this rank.
|
||||
num_org_vocab_padding: Padding rows between this rank's original and
|
||||
added vocab shards.
|
||||
added_vocab_start_index: First added vocab id owned by this rank.
|
||||
added_vocab_end_index: One past the last added vocab id owned by
|
||||
this rank.
|
||||
|
||||
Returns:
|
||||
``[num_tokens, embedding_dim]`` partial embeddings.
|
||||
|
||||
"""
|
||||
out = torch.empty(
|
||||
input_ids.shape[0],
|
||||
@@ -399,6 +406,7 @@ def apply_repetition_penalties(
|
||||
prompt_mask: A boolean tensor indicating which tokens appear in the prompt.
|
||||
output_mask: A boolean tensor indicating which tokens appear in the output.
|
||||
repetition_penalties: The repetition penalties of shape (num_seqs, ).
|
||||
|
||||
"""
|
||||
if logits.is_cuda and logits.is_contiguous():
|
||||
apply_repetition_penalties_cuda(
|
||||
@@ -787,8 +795,7 @@ def cutlass_scaled_mm(
|
||||
out_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
`cutlass_scaled_mm` implements a fused version of
|
||||
"""`cutlass_scaled_mm` implements a fused version of
|
||||
`output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
|
||||
where scale_a * a and scale_b * b are implemented using numpy-style
|
||||
broadcasting.
|
||||
@@ -840,11 +847,11 @@ def cutlass_scaled_mm_azp(
|
||||
azp: torch.Tensor | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
azp_adj: In the per-tensor case, this should include the azp.
|
||||
Always per-channel.
|
||||
azp: Only set in the per-token case. Per-token if set.
|
||||
"""Args:
|
||||
azp_adj: In the per-tensor case, this should include the azp.
|
||||
Always per-channel.
|
||||
azp: Only set in the per-token case. Per-token if set.
|
||||
|
||||
"""
|
||||
assert b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0
|
||||
assert out_dtype is torch.bfloat16 or out_dtype is torch.float16
|
||||
@@ -883,8 +890,7 @@ def get_cutlass_moe_mm_data(
|
||||
blockscale_offsets: torch.Tensor | None = None,
|
||||
is_gated: bool = True,
|
||||
):
|
||||
"""
|
||||
Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
"""Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
used in CUTLASS-based fused MoE.
|
||||
|
||||
The function takes in topk_ids (token-expert mapping) and uses it to
|
||||
@@ -943,8 +949,7 @@ def get_cutlass_moe_mm_problem_sizes_from_expert_offsets(
|
||||
|
||||
|
||||
def shuffle_rows(input_tensor: torch.Tensor, dst2src_map: torch.Tensor):
|
||||
"""
|
||||
Shuffle and expand the input tensor according to the dst2src_map and store the result in output_tensor.
|
||||
"""Shuffle and expand the input tensor according to the dst2src_map and store the result in output_tensor.
|
||||
This is used in MoE to permute the input tensor before performing grouped matrix multiplications.
|
||||
"""
|
||||
num_tokens_permuted = dst2src_map.shape[0]
|
||||
@@ -967,8 +972,7 @@ def get_cutlass_batched_moe_mm_data(
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""
|
||||
Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
"""Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
used in CUTLASS-based fused MoE.
|
||||
|
||||
The function takes in expert_num_tokens (token count per expert) and
|
||||
@@ -1006,8 +1010,7 @@ def cutlass_moe_mm(
|
||||
per_act_token: bool,
|
||||
per_out_ch: bool,
|
||||
):
|
||||
"""
|
||||
A single grouped matrix multiplication used in CUTLASS-based fused MoE.
|
||||
"""A single grouped matrix multiplication used in CUTLASS-based fused MoE.
|
||||
The function executes fp8-quantized OUT = AB matrix multiplication.
|
||||
|
||||
- expert_offsets: Indices that mark at which token index each expert begins
|
||||
@@ -1044,8 +1047,7 @@ def cutlass_fp4_moe_mm(
|
||||
expert_offsets: torch.Tensor,
|
||||
sf_offsets: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
|
||||
"""An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
|
||||
the gemms for each combination based on the specified problem sizes.
|
||||
|
||||
This is used as the MoE gemm during NVFP4 Quantized MoERunner forward.
|
||||
@@ -1083,8 +1085,7 @@ def cutlass_mxfp4_moe_mm(
|
||||
expert_offsets: torch.Tensor,
|
||||
sf_offsets: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
An MXFP4 Blockscaled Group Gemm for MoE (MXFP4 x MXFP4).
|
||||
"""An MXFP4 Blockscaled Group Gemm for MoE (MXFP4 x MXFP4).
|
||||
|
||||
Uses mx_float4_t types with E8M0 scale factors and 32-element blocks.
|
||||
- a/b_tensors: MXFP4 packed activations/weights (uint8, 2 E2M1 per byte)
|
||||
@@ -1458,8 +1459,7 @@ def cutlass_w4a8_moe_mm(
|
||||
group_scale_strides: torch.Tensor,
|
||||
maybe_schedule: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Executes the CUTLASS-based fused-MoE grouped matrix multiplication for the
|
||||
"""Executes the CUTLASS-based fused-MoE grouped matrix multiplication for the
|
||||
W4A8 quantization scheme. Uses group-wise quantization (INT4 -> FP8)
|
||||
and both per-channel + per-token scaling in the epilogue.
|
||||
|
||||
@@ -1495,6 +1495,7 @@ def cutlass_w4a8_moe_mm(
|
||||
|
||||
Returns:
|
||||
out_tensors updated in-place with the dequantized INT4xFP8 grouped GEMM result.
|
||||
|
||||
"""
|
||||
return torch.ops._C.cutlass_w4a8_moe_mm(
|
||||
out_tensors,
|
||||
@@ -1535,8 +1536,7 @@ def scaled_fp4_quant(
|
||||
backend: str = "none",
|
||||
padded_n: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP4 and return quantized tensor and scale.
|
||||
"""Quantize input tensor to FP4 and return quantized tensor and scale.
|
||||
|
||||
This function quantizes the last dimension of the given tensor `input`. For
|
||||
every 16 consecutive elements, a single dynamically computed scaling factor
|
||||
@@ -1559,6 +1559,7 @@ def scaled_fp4_quant(
|
||||
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every
|
||||
two values are packed into a uint8 and float8_e4m3 scaling factors
|
||||
in the sizzled layout.
|
||||
|
||||
"""
|
||||
assert not current_platform.is_rocm()
|
||||
assert input.ndim >= 1, f"input.ndim needs to be >= 1, but got {input.ndim}."
|
||||
@@ -1613,17 +1614,19 @@ def scaled_fp4_experts_quant(
|
||||
blockscale_offsets: torch.Tensor,
|
||||
topk: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to NVFP4 and return quantized tensor and scale, for
|
||||
"""Quantize input tensor to NVFP4 and return quantized tensor and scale, for
|
||||
packed MoE Inputs.
|
||||
|
||||
Args:
|
||||
input_tensor: The input tensor to be quantized to NVFP4
|
||||
input_global_scale: A scalar scaling factor for the entire tensor.
|
||||
expert_offsets: The expert offsets tensor
|
||||
blockscale_offsets: The blockscale offsets tensor
|
||||
topk: The number of experts each token is routed to
|
||||
Outputs:
|
||||
output: The quantized tensor in NVFP4
|
||||
output_scales: The blockscale tensor in FP8-E4M3
|
||||
|
||||
"""
|
||||
assert not current_platform.is_rocm()
|
||||
assert input_tensor.ndim == 2, (
|
||||
@@ -1675,8 +1678,7 @@ def silu_and_mul_scaled_fp4_experts_quant(
|
||||
blockscale_offsets: torch.Tensor,
|
||||
topk: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Fused SiLU+Mul+NVFP4 quantization for MoE intermediate activations.
|
||||
"""Fused SiLU+Mul+NVFP4 quantization for MoE intermediate activations.
|
||||
|
||||
Args:
|
||||
input_tensor: The input tensor with gate || up layout [m_topk, k*2]
|
||||
@@ -1687,6 +1689,7 @@ def silu_and_mul_scaled_fp4_experts_quant(
|
||||
Outputs:
|
||||
output: The quantized tensor in NVFP4 [m_topk, k/2]
|
||||
output_scales: The blockscale tensor in FP8-E4M3
|
||||
|
||||
"""
|
||||
assert not current_platform.is_rocm()
|
||||
assert input_tensor.ndim == 2, (
|
||||
@@ -1740,8 +1743,7 @@ def mxfp4_experts_quant(
|
||||
n_experts: int,
|
||||
topk: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to MXFP4 for packed MoE inputs.
|
||||
"""Quantize input tensor to MXFP4 for packed MoE inputs.
|
||||
Uses 32-element blocks with E8M0 (power-of-two) scale factors.
|
||||
MXFP4 has no global scale - only block-level E8M0 scale factors.
|
||||
|
||||
@@ -1754,6 +1756,7 @@ def mxfp4_experts_quant(
|
||||
Returns:
|
||||
output: [m_topk, k//2] packed E2M1 values (uint8)
|
||||
output_scales: E8M0 blockscales in swizzled layout (uint8 view)
|
||||
|
||||
"""
|
||||
assert not current_platform.is_rocm()
|
||||
assert input_tensor.ndim == 2
|
||||
@@ -1799,8 +1802,7 @@ def silu_and_mul_mxfp4_experts_quant(
|
||||
n_experts: int,
|
||||
topk: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Fused SiLU+Mul+MXFP4 quantization for MoE intermediate activations.
|
||||
"""Fused SiLU+Mul+MXFP4 quantization for MoE intermediate activations.
|
||||
MXFP4 has no global scale - only block-level E8M0 scale factors.
|
||||
"""
|
||||
assert not current_platform.is_rocm()
|
||||
@@ -1846,8 +1848,7 @@ def scaled_fp8_quant(
|
||||
output: torch.Tensor | None = None,
|
||||
group_shape: tuple[int, int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
"""Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
|
||||
This function supports both static and dynamic quantization: If you
|
||||
provide the scale, it will use static scaling and if you omit it,
|
||||
@@ -1869,6 +1870,8 @@ def scaled_fp8_quant(
|
||||
of the output to at least this value.
|
||||
use_per_token_if_dynamic: Whether to do per_tensor or per_token
|
||||
in the dynamic quantization case.
|
||||
output: Optional tensor to write the quantized result into. A new
|
||||
tensor is allocated when omitted.
|
||||
group_shape: Optional tuple (group_m, group_n) specifying the group
|
||||
shape for static quantization. Use -1 for "full extent" (e.g.,
|
||||
(-1, -1) for per-tensor, (-1, 1) for per-channel, etc.)
|
||||
@@ -1877,6 +1880,7 @@ def scaled_fp8_quant(
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
|
||||
scaling factor.
|
||||
|
||||
"""
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
@@ -1913,8 +1917,7 @@ def allspark_repack_weight(
|
||||
zero_point: torch.Tensor | None = None,
|
||||
has_zp: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format
|
||||
"""Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format
|
||||
for Ampere W8A16 Fused Gemm kernel
|
||||
|
||||
Args:
|
||||
@@ -1928,6 +1931,7 @@ def allspark_repack_weight(
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None] :
|
||||
rearranged weight, scale, and optionally zero_point.
|
||||
|
||||
"""
|
||||
K = qweight.shape[0]
|
||||
N = qweight.shape[1]
|
||||
@@ -1997,8 +2001,7 @@ def scaled_int8_quant(
|
||||
azp: torch.Tensor | None = None,
|
||||
symmetric: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
|
||||
"""Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
|
||||
|
||||
Args:
|
||||
input: The input tensor to be quantized to int8.
|
||||
@@ -2010,6 +2013,7 @@ def scaled_int8_quant(
|
||||
|
||||
Returns:
|
||||
Output int8 tensor, scales, and optionally azp.
|
||||
|
||||
"""
|
||||
if current_platform.is_xpu():
|
||||
# XPU has no _C int8 quant op; use the torch.compile reference.
|
||||
@@ -2463,8 +2467,7 @@ def grouped_topk(
|
||||
bias: torch.Tensor,
|
||||
scoring_func: int = 0,
|
||||
):
|
||||
"""
|
||||
Perform grouped top-k routing for mixture of experts.
|
||||
"""Perform grouped top-k routing for mixture of experts.
|
||||
|
||||
Args:
|
||||
scores: Raw inputs (logits if scoring_func=1, scores if scoring_func=0)
|
||||
@@ -2475,6 +2478,7 @@ def grouped_topk(
|
||||
routed_scaling_factor: Scaling factor for routing weights
|
||||
bias: Bias tensor (e_score_correction_bias). Always fused in kernel.
|
||||
scoring_func: 0=none (no activation), 1=sigmoid
|
||||
|
||||
"""
|
||||
if not (current_platform.is_cuda() or current_platform.is_xpu()):
|
||||
raise NotImplementedError(
|
||||
@@ -2911,8 +2915,7 @@ def swap_blocks(
|
||||
block_size_in_bytes: int,
|
||||
block_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Copy specific blocks from one tensor to another.
|
||||
"""Copy specific blocks from one tensor to another.
|
||||
|
||||
This method assumes each of the two input tensors is composed of
|
||||
consecutive contiguous blocks, of size block_size_in_bytes.
|
||||
@@ -2940,8 +2943,7 @@ def swap_blocks_batch(
|
||||
sizes: torch.Tensor,
|
||||
is_src_access_order_any: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Batch version of swap_blocks: submit all copies in a single driver call.
|
||||
"""Batch version of swap_blocks: submit all copies in a single driver call.
|
||||
|
||||
Each entry specifies a raw pointer copy: src_ptrs[i] -> dst_ptrs[i]
|
||||
of sizes[i] bytes. All three tensors must be CPU tensors with the
|
||||
@@ -3030,6 +3032,7 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
||||
host_cache: Optional pinned host rows used for non-resident entries
|
||||
host_row_ids: Host source row for each remapped cache row
|
||||
device_row_ids: Device source row, or -1, for each remapped cache row
|
||||
|
||||
"""
|
||||
torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache,
|
||||
@@ -3059,6 +3062,7 @@ def cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||
block_table: Block indices [num_reqs, max_blocks]
|
||||
workspace_starts: Workspace start offsets [num_reqs]
|
||||
batch_size: Number of requests
|
||||
|
||||
"""
|
||||
torch.ops._C_cache_ops.cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||
src_cache,
|
||||
@@ -3080,6 +3084,7 @@ def concat_mla_q(
|
||||
ql_nope: Query nope component [num_tokens, num_heads, nope_dim]
|
||||
q_pe: Query rope component [num_tokens, num_heads, rope_dim]
|
||||
q_out: Output tensor [num_tokens, num_heads, nope_dim + rope_dim]
|
||||
|
||||
"""
|
||||
torch.ops._C_cache_ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
@@ -3802,8 +3807,7 @@ def onednn_scaled_int8_quant(
|
||||
azp: torch.Tensor | None = None,
|
||||
symmetric: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
|
||||
"""Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
|
||||
|
||||
Args:
|
||||
input: The input tensor to be quantized to int8.
|
||||
@@ -3815,6 +3819,7 @@ def onednn_scaled_int8_quant(
|
||||
|
||||
Returns:
|
||||
Output int8 tensor, scales, and optionally azp.
|
||||
|
||||
"""
|
||||
output = torch.empty_like(input, dtype=torch.int8)
|
||||
token_num = input.numel() // input.shape[-1]
|
||||
@@ -4674,8 +4679,7 @@ def safeFusedQuantizeNv(
|
||||
xh_e4m3: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
|
||||
"""Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
|
||||
rather than returning them, to prevent torch 2.12+ errors that outputs of custom
|
||||
operators may not alias any inputs to the custom operator.
|
||||
"""
|
||||
@@ -4684,8 +4688,7 @@ def safeFusedQuantizeNv(
|
||||
|
||||
|
||||
def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
|
||||
"""
|
||||
Perform Hadamard transforms using [Hadacore](https://arxiv.org/abs/2412.08832)
|
||||
"""Perform Hadamard transforms using [Hadacore](https://arxiv.org/abs/2412.08832)
|
||||
kernels. Note that these kernels exploit the recursive properties of
|
||||
Sylvester Hadamards, and therefore do not require transform weight data
|
||||
|
||||
@@ -4698,6 +4701,7 @@ def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
|
||||
|
||||
Returns:
|
||||
value after transformation
|
||||
|
||||
"""
|
||||
return torch.ops._C.hadacore_transform(x, inplace)
|
||||
|
||||
|
||||
@@ -286,6 +286,7 @@ def _xpu_fp8_bmm_impl(
|
||||
This implementation centralizes access to
|
||||
``torch.ops._xpu_C.fp8_bmm``. Both scales must be contiguous, while
|
||||
``a`` and ``b`` may be non-contiguous views.
|
||||
|
||||
"""
|
||||
return torch.ops._xpu_C.fp8_bmm(a, b, out_dtype, a_scale, b_scale, bias)
|
||||
|
||||
@@ -430,6 +431,7 @@ def _xpu_deepseek_fused_indexer_q_rope_fp8_impl(
|
||||
index_weights_out: (T, H) float32 output, preallocated;
|
||||
= index_weights * q_scale * softmax_scale * head_scale (the
|
||||
per-(token, head) q_scale is folded in here). [written]
|
||||
|
||||
"""
|
||||
torch.ops._xpu_C.deepseek_fused_indexer_q_rope_fp8(
|
||||
index_q,
|
||||
@@ -473,6 +475,7 @@ def _xpu_deepseek_fused_indexer_q_rope_mxfp4_impl(
|
||||
index_weights_out: (T, H) float32 output, preallocated;
|
||||
= index_weights * softmax_scale * head_scale (no q_scale folded;
|
||||
per-block scales live in index_q_scale). [written]
|
||||
|
||||
"""
|
||||
torch.ops._xpu_C.deepseek_fused_indexer_q_rope_mxfp4(
|
||||
index_q,
|
||||
|
||||
+1
-2
@@ -20,8 +20,7 @@ def get_cache_dir() -> Path:
|
||||
|
||||
@lru_cache
|
||||
def get_vllm_public_assets(filename: str, s3_prefix: str | None = None) -> Path:
|
||||
"""
|
||||
Download an asset file from `s3://vllm-public-assets`
|
||||
"""Download an asset file from `s3://vllm-public-assets`
|
||||
and return the path to the downloaded file.
|
||||
"""
|
||||
asset_directory = get_cache_dir() / "vllm_public_assets"
|
||||
|
||||
@@ -34,9 +34,7 @@ class ImageAsset:
|
||||
name: ImageAssetName
|
||||
|
||||
def get_path(self, ext: str) -> Path:
|
||||
"""
|
||||
Return s3 path for given image.
|
||||
"""
|
||||
"""Return s3 path for given image."""
|
||||
return get_vllm_public_assets(
|
||||
filename=f"{self.name}.{ext}", s3_prefix=VLM_IMAGES_DIR
|
||||
)
|
||||
@@ -51,9 +49,7 @@ class ImageAsset:
|
||||
|
||||
@property
|
||||
def image_embeds(self) -> torch.Tensor:
|
||||
"""
|
||||
Image embeddings, only used for testing purposes with llava 1.5.
|
||||
"""
|
||||
"""Image embeddings, only used for testing purposes with llava 1.5."""
|
||||
image_path = self.get_path("pt")
|
||||
return torch.load(image_path, map_location="cpu", weights_only=True)
|
||||
|
||||
|
||||
@@ -21,8 +21,7 @@ def _sample_frame_indices(total_frames: int, num_frames: int) -> npt.NDArray:
|
||||
|
||||
@lru_cache
|
||||
def download_video_asset(filename: str) -> str:
|
||||
"""
|
||||
Download and open an image from huggingface
|
||||
"""Download and open an image from huggingface
|
||||
repo: raushan-testing-hf/videos-test
|
||||
"""
|
||||
video_directory = get_cache_dir() / "video-example-data"
|
||||
@@ -141,8 +140,7 @@ class VideoAsset:
|
||||
return ret
|
||||
|
||||
def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray:
|
||||
"""
|
||||
Read audio data from the video asset, used in Qwen2.5-Omni examples.
|
||||
"""Read audio data from the video asset, used in Qwen2.5-Omni examples.
|
||||
|
||||
See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
"""Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
compatible with ``CustomDataset`` (``--dataset-name custom``), by
|
||||
randomly slicing the tokenized text into prompts.
|
||||
|
||||
@@ -66,8 +65,8 @@ def create_txt_slices_jsonl(
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
"""Read *input_path*, slice it into prompts, and write JSONL to
|
||||
*output_path*."""
|
||||
|
||||
*output_path*.
|
||||
"""
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name, trust_remote_code=trust_remote_code
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This module defines a framework for sampling benchmark requests from various
|
||||
"""This module defines a framework for sampling benchmark requests from various
|
||||
datasets. Each dataset subclass of BenchmarkDataset must implement sample
|
||||
generation. Supported dataset types include:
|
||||
- ShareGPT
|
||||
@@ -75,9 +74,7 @@ DEFAULT_NUM_PROMPTS = 1000
|
||||
|
||||
@dataclass
|
||||
class SampleRequest:
|
||||
"""
|
||||
Represents a single inference request for benchmarking.
|
||||
"""
|
||||
"""Represents a single inference request for benchmarking."""
|
||||
|
||||
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
|
||||
prompt_len: int
|
||||
@@ -112,8 +109,7 @@ class BenchmarkDataset(ABC):
|
||||
disable_shuffle: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the BenchmarkDataset with an optional dataset path and random
|
||||
"""Initialize the BenchmarkDataset with an optional dataset path and random
|
||||
seed.
|
||||
|
||||
Args:
|
||||
@@ -121,6 +117,9 @@ class BenchmarkDataset(ABC):
|
||||
indicates that a default or random dataset might be used.
|
||||
random_seed (int): Seed value for reproducible shuffling or
|
||||
sampling. Defaults to DEFAULT_SEED.
|
||||
disable_shuffle (bool): If True, keep the dataset in its original
|
||||
order instead of shuffling it.
|
||||
|
||||
"""
|
||||
self.dataset_path = dataset_path
|
||||
# Set the random seed, ensuring that a None value is replaced with the
|
||||
@@ -134,8 +133,7 @@ class BenchmarkDataset(ABC):
|
||||
prompt: str,
|
||||
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Transform a prompt and optional multimodal content into a chat format.
|
||||
"""Transform a prompt and optional multimodal content into a chat format.
|
||||
This method is used for chat models that expect a specific conversation
|
||||
format.
|
||||
"""
|
||||
@@ -152,14 +150,14 @@ class BenchmarkDataset(ABC):
|
||||
return [{"role": "user", "content": content}]
|
||||
|
||||
def load_data(self) -> None:
|
||||
"""
|
||||
Load data from the dataset path into self.data.
|
||||
"""Load data from the dataset path into self.data.
|
||||
|
||||
This method must be overridden by subclasses since the method to load
|
||||
data will vary depending on the dataset format and source.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If a subclass does not implement this method.
|
||||
|
||||
"""
|
||||
# TODO (jenniferzhao): add support for downloading data
|
||||
raise NotImplementedError("load_data must be implemented in subclasses.")
|
||||
@@ -169,8 +167,7 @@ class BenchmarkDataset(ABC):
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Optionally select a random LoRA request.
|
||||
"""Optionally select a random LoRA request.
|
||||
|
||||
This method is used when LoRA parameters are provided. It randomly
|
||||
selects a LoRA based on max_loras.
|
||||
@@ -184,6 +181,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if max_loras is None or lora_path is None:
|
||||
return None
|
||||
@@ -203,8 +201,7 @@ class BenchmarkDataset(ABC):
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Optionally select a LoRA request using deterministic round-robin.
|
||||
"""Optionally select a LoRA request using deterministic round-robin.
|
||||
|
||||
This method cycles through LoRA IDs in order based on the request
|
||||
index, providing reproducible LoRA assignment.
|
||||
@@ -219,6 +216,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if max_loras is None or lora_path is None:
|
||||
return None
|
||||
@@ -239,8 +237,7 @@ class BenchmarkDataset(ABC):
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Select a LoRA request using the specified assignment strategy.
|
||||
"""Select a LoRA request using the specified assignment strategy.
|
||||
|
||||
Args:
|
||||
index (int): The request index (used for round-robin).
|
||||
@@ -252,6 +249,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if lora_assignment == "round-robin":
|
||||
return BenchmarkDataset.get_round_robin_lora_request(
|
||||
@@ -270,8 +268,7 @@ class BenchmarkDataset(ABC):
|
||||
no_oversample: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
"""
|
||||
Abstract method to generate sample requests from the dataset.
|
||||
"""Abstract method to generate sample requests from the dataset.
|
||||
|
||||
Subclasses must override this method to implement dataset-specific logic
|
||||
for generating a list of SampleRequest objects.
|
||||
@@ -281,10 +278,13 @@ class BenchmarkDataset(ABC):
|
||||
for processing the dataset's text.
|
||||
num_requests (int): The number of sample requests to generate.
|
||||
request_id_prefix (str): The prefix of request_id.
|
||||
no_oversample (bool): If True, do not oversample the dataset to
|
||||
reach num_requests.
|
||||
|
||||
Returns:
|
||||
list[SampleRequest]: A list of sample requests generated from the
|
||||
dataset.
|
||||
|
||||
"""
|
||||
raise NotImplementedError("sample must be implemented in subclasses.")
|
||||
|
||||
@@ -295,8 +295,7 @@ class BenchmarkDataset(ABC):
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Oversamples the list of requests if its size is less than the desired
|
||||
"""Oversamples the list of requests if its size is less than the desired
|
||||
number.
|
||||
|
||||
Args:
|
||||
@@ -305,6 +304,8 @@ class BenchmarkDataset(ABC):
|
||||
num_requests (int): The target number of requests.
|
||||
request_id_prefix (str): The prefix applied to generated request
|
||||
identifiers.
|
||||
no_oversample (bool): If True, return the requests unchanged
|
||||
instead of oversampling up to num_requests.
|
||||
|
||||
"""
|
||||
if no_oversample:
|
||||
@@ -346,8 +347,7 @@ def is_valid_sequence(
|
||||
max_total_len: int = 2048,
|
||||
skip_min_output_len_check: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Validate a sequence based on prompt and output lengths.
|
||||
"""Validate a sequence based on prompt and output lengths.
|
||||
|
||||
Default pruning criteria are copied from the original `sample_hf_requests`
|
||||
and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
|
||||
@@ -379,8 +379,7 @@ def process_image(
|
||||
*,
|
||||
ensure_client_side_data: bool = False,
|
||||
) -> Mapping[str, Any]:
|
||||
"""
|
||||
Process a single image input and return a multimedia content dictionary.
|
||||
"""Process a single image input and return a multimedia content dictionary.
|
||||
|
||||
Supports the following input types:
|
||||
|
||||
@@ -402,6 +401,7 @@ def process_image(
|
||||
|
||||
Raises:
|
||||
ValueError: If the input is not a supported type.
|
||||
|
||||
"""
|
||||
if isinstance(image, dict) and "bytes" in image:
|
||||
image = Image.open(BytesIO(image["bytes"]))
|
||||
@@ -437,8 +437,7 @@ def process_image(
|
||||
|
||||
|
||||
def process_video(video: Any) -> Mapping[str, Any]:
|
||||
"""
|
||||
Process a single video input and return a multimedia content dictionary.
|
||||
"""Process a single video input and return a multimedia content dictionary.
|
||||
|
||||
Supports the following input types:
|
||||
|
||||
@@ -451,6 +450,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
|
||||
|
||||
Raises:
|
||||
ValueError: If the input is not a supported type.
|
||||
|
||||
"""
|
||||
if isinstance(video, dict) and "bytes" in video:
|
||||
video_bytes = video["bytes"]
|
||||
@@ -474,8 +474,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
|
||||
|
||||
|
||||
def process_audio(audio: Any) -> tuple:
|
||||
"""
|
||||
Process a single audio input and return a (array, sample_rate) tuple.
|
||||
"""Process a single audio input and return a (array, sample_rate) tuple.
|
||||
|
||||
Supports:
|
||||
1. String: treated as a file path, loaded with soundfile.
|
||||
@@ -502,8 +501,7 @@ def gen_prompt_decode_to_target_len(
|
||||
add_special_tokens: bool = False,
|
||||
rng: np.random.Generator | None = None,
|
||||
) -> tuple[str, list[int], int]:
|
||||
"""
|
||||
Ensure decoded-then-encoded prompt length matches the target token length.
|
||||
"""Ensure decoded-then-encoded prompt length matches the target token length.
|
||||
|
||||
This function decodes an initial token sequence to text and re-encodes it
|
||||
, iteratively adjusting the token sequence length to match a target.
|
||||
@@ -557,8 +555,7 @@ def gen_prompt_decode_to_target_len(
|
||||
|
||||
|
||||
class RandomDataset(BenchmarkDataset):
|
||||
"""
|
||||
Synthetic text-only dataset for serving/throughput benchmarks.
|
||||
"""Synthetic text-only dataset for serving/throughput benchmarks.
|
||||
|
||||
Strategy:
|
||||
- Sample input/output token lengths per request from integer-uniform ranges
|
||||
@@ -699,9 +696,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
allowed_tokens: np.ndarray,
|
||||
prefix_len: int,
|
||||
) -> list[int]:
|
||||
"""
|
||||
Get the prefix for the dataset.
|
||||
"""
|
||||
"""Get the prefix for the dataset."""
|
||||
if prefix_len <= 0:
|
||||
return []
|
||||
|
||||
@@ -738,8 +733,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
index: int,
|
||||
allowed_tokens: np.ndarray,
|
||||
) -> tuple[str, int, int]:
|
||||
"""
|
||||
Returns (prompt, total_input_len).
|
||||
"""Returns (prompt, total_input_len).
|
||||
|
||||
NOTE: After decoding the prompt we have to encode and decode it again.
|
||||
This is done because in some cases N consecutive tokens
|
||||
@@ -778,8 +772,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class RandomDatasetForReranking(RandomDataset):
|
||||
"""
|
||||
Random dataset specialized for the needs of scoring:
|
||||
"""Random dataset specialized for the needs of scoring:
|
||||
- Batches of inputs
|
||||
- Inputs composed of pairs
|
||||
"""
|
||||
@@ -906,8 +899,7 @@ class RandomDatasetForReranking(RandomDataset):
|
||||
|
||||
|
||||
class RandomMultiModalDataset(RandomDataset):
|
||||
"""
|
||||
Synthetic multimodal dataset (text + images) that extends RandomDataset.
|
||||
"""Synthetic multimodal dataset (text + images) that extends RandomDataset.
|
||||
|
||||
Status:
|
||||
- Images: supported via synthetic RGB data.
|
||||
@@ -1022,8 +1014,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
def normalize_bucket_config(
|
||||
self, bucket_config: dict[tuple[int, int, int], float]
|
||||
) -> dict[tuple[int, int, int], float]:
|
||||
"""
|
||||
Remove zero probability entries
|
||||
"""Remove zero probability entries
|
||||
and normalize the bucket config to sum to 1.
|
||||
"""
|
||||
# Raise error if value is negative
|
||||
@@ -1044,13 +1035,11 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
self,
|
||||
mm_item_config: tuple[int, int, int],
|
||||
) -> Mapping[str, Any]:
|
||||
"""
|
||||
Create synthetic images and videos and
|
||||
"""Create synthetic images and videos and
|
||||
apply process_image/process_video respectively.
|
||||
This follows the OpenAI API chat completions
|
||||
https://github.com/openai/openai-python
|
||||
"""
|
||||
|
||||
if self.map_config_to_modality(mm_item_config) == "image":
|
||||
return process_image(
|
||||
self.generate_synthetic_image(mm_item_config[1], mm_item_config[0])
|
||||
@@ -1071,9 +1060,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
bucket_config: dict[tuple[int, int, int], float],
|
||||
) -> tuple[int, int, dict[str, int], dict[tuple[int, int, int], float]]:
|
||||
"""
|
||||
Get the sampling parameters for the multimodal items.
|
||||
"""
|
||||
"""Get the sampling parameters for the multimodal items."""
|
||||
# Enforce num_mm_items_range_ratio <= 1
|
||||
if not (0.0 <= num_mm_items_range_ratio <= 1.0):
|
||||
raise ValueError("num_mm_items_range_ratio must be in [0, 1].")
|
||||
@@ -1146,8 +1133,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
bucket_config: dict[tuple[int, int, int], float],
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
) -> Iterator[tuple[int, int, int]]:
|
||||
"""
|
||||
Iterator over the multimodal items for each request
|
||||
"""Iterator over the multimodal items for each request
|
||||
whose size is between min_num_mm_items and max_num_mm_items.
|
||||
|
||||
Loop over the bucket config and sample a multimodal item.
|
||||
@@ -1160,6 +1146,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
`bucket_config` (tuple->float). The original dict passed to
|
||||
`sample` is not mutated. If this ever changes, a test
|
||||
is implemented and will fail.
|
||||
|
||||
"""
|
||||
# Get the number of multimodal items to sample
|
||||
request_num_mm_items = int(
|
||||
@@ -1346,8 +1333,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
|
||||
|
||||
class ShareGPTDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the ShareGPT dataset. Loads data from a JSON file and generates
|
||||
"""Implements the ShareGPT dataset. Loads data from a JSON file and generates
|
||||
sample requests based on conversation turns.
|
||||
"""
|
||||
|
||||
@@ -1437,8 +1423,7 @@ class ShareGPTDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class TimedTrace(BenchmarkDataset):
|
||||
"""
|
||||
Implements a base class to replay various timed traces.
|
||||
"""Implements a base class to replay various timed traces.
|
||||
Loads data from a JSON file and generates sample requests
|
||||
based on the timing information in the traces.
|
||||
"""
|
||||
@@ -1927,6 +1912,7 @@ def add_random_dataset_base_args(
|
||||
|
||||
Args:
|
||||
parser_or_group: Either a parser or an argument group to add arguments to.
|
||||
|
||||
"""
|
||||
parser_or_group.add_argument(
|
||||
"--random-input-len",
|
||||
@@ -1988,6 +1974,7 @@ def add_random_multimodal_dataset_args(
|
||||
|
||||
Args:
|
||||
parser_or_group: Either a parser or an argument group to add arguments to.
|
||||
|
||||
"""
|
||||
parser_or_group.add_argument(
|
||||
"--random-mm-base-items-per-request",
|
||||
@@ -2503,8 +2490,7 @@ def get_samples(
|
||||
|
||||
|
||||
class CustomDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the Custom dataset. Loads data from a JSONL file and generates
|
||||
"""Implements the Custom dataset. Loads data from a JSONL file and generates
|
||||
sample requests based on conversation turns. E.g.,
|
||||
```
|
||||
{"prompt": "What is the capital of India?", "output_tokens": 10}
|
||||
@@ -2632,8 +2618,7 @@ class CustomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class CustomImageDataset(CustomDataset):
|
||||
"""
|
||||
Implements the Custom image dataset. Loads data from a JSONL file and generates
|
||||
"""Implements the Custom image dataset. Loads data from a JSONL file and generates
|
||||
sample requests based on conversation turns. E.g.,
|
||||
```
|
||||
{
|
||||
@@ -2917,8 +2902,7 @@ class CustomImageDataset(CustomDataset):
|
||||
|
||||
|
||||
class CustomAudioDataset(CustomDataset):
|
||||
"""
|
||||
Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
|
||||
"""Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
|
||||
{"prompt": "Transcribe the audio.", "audio": "/path/to/audio.wav"}
|
||||
|
||||
Supports both:
|
||||
@@ -3027,8 +3011,7 @@ class CustomAudioDataset(CustomDataset):
|
||||
|
||||
|
||||
class SpecBench(CustomDataset):
|
||||
"""
|
||||
Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
|
||||
"""Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
|
||||
Download the dataset using:
|
||||
wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl
|
||||
""" # noqa: E501
|
||||
@@ -3100,8 +3083,7 @@ class SpecBench(CustomDataset):
|
||||
"SonnetDataset is deprecated and will be removed in a future version.",
|
||||
)
|
||||
class SonnetDataset(BenchmarkDataset):
|
||||
"""
|
||||
Simplified implementation of the Sonnet dataset. Loads poem lines from a
|
||||
"""Simplified implementation of the Sonnet dataset. Loads poem lines from a
|
||||
text file and generates sample requests. Default values here copied from
|
||||
`benchmark_serving.py` for the sonnet dataset.
|
||||
"""
|
||||
@@ -3198,8 +3180,7 @@ class SonnetDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class BurstGPTDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the BurstGPT dataset. Loads data from a CSV file and generates
|
||||
"""Implements the BurstGPT dataset. Loads data from a CSV file and generates
|
||||
sample requests based on synthetic prompt generation. Only rows with Model
|
||||
"GPT-4" and positive response tokens are used.
|
||||
"""
|
||||
@@ -3447,9 +3428,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class VisionArenaDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Vision Arena Dataset.
|
||||
"""
|
||||
"""Vision Arena Dataset."""
|
||||
|
||||
DEFAULT_OUTPUT_LEN = 128
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
@@ -3506,8 +3485,7 @@ class VisionArenaDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MMVUDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MMVU Dataset.
|
||||
"""MMVU Dataset.
|
||||
https://huggingface.co/datasets/yale-nlp/MMVU
|
||||
"""
|
||||
|
||||
@@ -3586,8 +3564,7 @@ class MMVUDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class InstructCoderDataset(HuggingFaceDataset):
|
||||
"""
|
||||
InstructCoder Dataset.
|
||||
"""InstructCoder Dataset.
|
||||
https://huggingface.co/datasets/likaixin/InstructCoder
|
||||
|
||||
InstructCoder is the dataset designed for general code editing. It consists
|
||||
@@ -3655,8 +3632,7 @@ class InstructCoderDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MTBenchDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MT-Bench Dataset.
|
||||
"""MT-Bench Dataset.
|
||||
https://huggingface.co/datasets/philschmid/mt-bench
|
||||
|
||||
We create a single turn dataset for MT-Bench.
|
||||
@@ -3718,8 +3694,7 @@ class MTBenchDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class HumanEvalDataset(HuggingFaceDataset):
|
||||
"""
|
||||
HumanEvalDataset Dataset.
|
||||
"""HumanEvalDataset Dataset.
|
||||
https://huggingface.co/datasets/openai/openai_humaneval
|
||||
|
||||
We create a single turn dataset for HumanEval.
|
||||
@@ -3779,8 +3754,7 @@ class HumanEvalDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class GSM8KDataset(HuggingFaceDataset):
|
||||
"""
|
||||
GSM8K Dataset.
|
||||
"""GSM8K Dataset.
|
||||
https://huggingface.co/datasets/openai/gsm8k
|
||||
|
||||
We create a single turn dataset for GSM8K.
|
||||
@@ -3840,8 +3814,7 @@ class GSM8KDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class BlazeditDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Blazedit Dataset.
|
||||
"""Blazedit Dataset.
|
||||
https://github.com/ise-uiuc/blazedit
|
||||
|
||||
5k char version: vdaita/edit_5k_char
|
||||
@@ -3932,9 +3905,7 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
|
||||
|
||||
|
||||
class AIMODataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a AIMO dataset with reasoning questions.
|
||||
"""
|
||||
"""Dataset class for processing a AIMO dataset with reasoning questions."""
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"AI-MO/aimo-validation-aime",
|
||||
@@ -4026,6 +3997,7 @@ def _format_zeta_prompt(
|
||||
|
||||
Returns:
|
||||
A dictionary with the formatted prompts and expected outputs.
|
||||
|
||||
"""
|
||||
events = sample["events"]
|
||||
input = sample["input"]
|
||||
@@ -4042,9 +4014,7 @@ def _format_zeta_prompt(
|
||||
|
||||
|
||||
class NextEditPredictionDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a Next Edit Prediction dataset.
|
||||
"""
|
||||
"""Dataset class for processing a Next Edit Prediction dataset."""
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"zed-industries/zeta",
|
||||
@@ -4092,8 +4062,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class ASRDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a ASR dataset for transcription.
|
||||
"""Dataset class for processing a ASR dataset for transcription.
|
||||
Tested on the following set:
|
||||
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
@@ -4290,8 +4259,7 @@ class ASRDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MLPerfDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MLPerf Inference Dataset.
|
||||
"""MLPerf Inference Dataset.
|
||||
|
||||
Dataset on HF:
|
||||
https://huggingface.co/datasets/mgoin/mlperf-inference-llama2-data
|
||||
@@ -4415,7 +4383,8 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
|
||||
|
||||
def _generate_exact_length_tokens(target_length: int) -> tuple[list[int], int]:
|
||||
"""Generate tokens that decode and re-encode to exactly
|
||||
target_length."""
|
||||
target_length.
|
||||
"""
|
||||
# Generate random tokens
|
||||
tokens = np.random.randint(0, vocab_size, size=target_length).tolist()
|
||||
|
||||
@@ -4470,8 +4439,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class MMStarDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
|
||||
"""Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
|
||||
refer to: https://github.com/sgl-project/SpecForge/pull/106
|
||||
"""
|
||||
|
||||
@@ -4760,8 +4728,7 @@ class BFCLDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class SpeedBench(CustomDataset):
|
||||
"""
|
||||
SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench
|
||||
"""SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench.
|
||||
|
||||
Download the dataset using:
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared utilities for benchmark dataset sampling.
|
||||
"""
|
||||
"""Shared utilities for benchmark dataset sampling."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
@@ -46,8 +44,7 @@ def get_sampling_params(
|
||||
output_len: int,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Sample per-request input/output token lengths and vocab offsets.
|
||||
"""Sample per-request input/output token lengths and vocab offsets.
|
||||
|
||||
Lengths are drawn uniformly from integer ranges around the configured
|
||||
means, controlled by *range_ratio*. It may be a single ``float``
|
||||
@@ -60,6 +57,7 @@ def get_sampling_params(
|
||||
Returns:
|
||||
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
|
||||
shape ``(num_requests,)``.
|
||||
|
||||
"""
|
||||
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
|
||||
|
||||
|
||||
@@ -22,7 +22,8 @@ AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
|
||||
|
||||
class StreamedResponseHandler:
|
||||
"""Handles streaming HTTP responses by accumulating chunks until complete
|
||||
messages are available."""
|
||||
messages are available.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.buffer = ""
|
||||
@@ -30,7 +31,8 @@ class StreamedResponseHandler:
|
||||
|
||||
def add_chunk(self, chunk_bytes: bytes) -> list[str]:
|
||||
"""Add a chunk of bytes to the buffer and return any complete
|
||||
messages."""
|
||||
messages.
|
||||
"""
|
||||
chunk_str = self._decoder.decode(chunk_bytes)
|
||||
self.buffer += chunk_str
|
||||
|
||||
@@ -167,10 +169,12 @@ async def async_request_openai_completions(
|
||||
|
||||
Args:
|
||||
request_func_input: The input for the request function.
|
||||
session: The aiohttp session used to issue the request.
|
||||
pbar: The progress bar to display the progress.
|
||||
|
||||
Returns:
|
||||
The output of the request function.
|
||||
|
||||
"""
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Completions API", "completions")
|
||||
|
||||
@@ -22,12 +22,12 @@ async def wait_for_endpoint(
|
||||
timeout_seconds: int = 600,
|
||||
retry_interval: int = 5,
|
||||
) -> RequestFuncOutput:
|
||||
"""
|
||||
Wait for an endpoint to become available before starting benchmarks.
|
||||
"""Wait for an endpoint to become available before starting benchmarks.
|
||||
|
||||
Args:
|
||||
request_func: The async request function to call
|
||||
test_input: The RequestFuncInput to test with
|
||||
session: The aiohttp session used to issue the probe requests
|
||||
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
|
||||
retry_interval: Time between retries in seconds (default: 5 seconds)
|
||||
|
||||
@@ -36,6 +36,7 @@ async def wait_for_endpoint(
|
||||
|
||||
Raises:
|
||||
ValueError: If the endpoint doesn't become available within the timeout
|
||||
|
||||
"""
|
||||
deadline = time.perf_counter() + timeout_seconds
|
||||
output = RequestFuncOutput(success=False)
|
||||
|
||||
@@ -36,9 +36,7 @@ def extract_field(
|
||||
|
||||
|
||||
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if the benchmark is run with torch.compile
|
||||
"""
|
||||
"""Check if the benchmark is run with torch.compile."""
|
||||
return not (
|
||||
extract_field(args, extra_info, "compilation_config.mode") == "0"
|
||||
or "eager" in getattr(args, "output_json", "")
|
||||
@@ -49,8 +47,7 @@ def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
|
||||
def convert_to_pytorch_benchmark_format(
|
||||
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
|
||||
) -> list:
|
||||
"""
|
||||
Save the benchmark results in the format used by PyTorch OSS benchmark with
|
||||
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
|
||||
on metric per record
|
||||
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
|
||||
"""
|
||||
|
||||
@@ -44,8 +44,7 @@ else:
|
||||
|
||||
|
||||
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
|
||||
"""
|
||||
Get all multimodal timing stats from the LLM engine.
|
||||
"""Get all multimodal timing stats from the LLM engine.
|
||||
|
||||
Collects both preprocessing stats (HF processor, hashing, cache lookup,
|
||||
prompt update) and encoder forward pass timing, merged by request_id.
|
||||
@@ -70,6 +69,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
|
||||
'num_encoder_calls': 1
|
||||
}
|
||||
}
|
||||
|
||||
"""
|
||||
observability_config = llm_engine.vllm_config.observability_config
|
||||
if not observability_config or not observability_config.enable_mm_processor_stats:
|
||||
@@ -126,8 +126,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
|
||||
|
||||
|
||||
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
|
||||
"""
|
||||
Collect multimodal processor timing stats.
|
||||
"""Collect multimodal processor timing stats.
|
||||
Returns a dictionary mapping stage names to lists of timing values (in seconds).
|
||||
"""
|
||||
all_stats = get_timing_stats_from_engine(llm_engine)
|
||||
@@ -147,9 +146,7 @@ def calculate_mm_processor_metrics(
|
||||
*,
|
||||
unit: Literal["us", "ms", "s"] = "ms",
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""
|
||||
Calculate aggregate metrics from stats by stage.
|
||||
"""
|
||||
"""Calculate aggregate metrics from stats by stage."""
|
||||
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
|
||||
unit_mult = unit2mult[unit]
|
||||
|
||||
@@ -181,9 +178,7 @@ def calculate_mm_processor_metrics(
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""
|
||||
Validate command-line arguments for mm_processor benchmark.
|
||||
"""
|
||||
"""Validate command-line arguments for mm_processor benchmark."""
|
||||
if not getattr(args, "tokenizer", None):
|
||||
args.tokenizer = args.model
|
||||
if not hasattr(args, "dataset_path"):
|
||||
@@ -214,9 +209,7 @@ def validate_args(args):
|
||||
def benchmark_multimodal_processor(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the multimodal processor benchmark.
|
||||
"""
|
||||
"""Run the multimodal processor benchmark."""
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
validate_args(args)
|
||||
@@ -455,7 +448,6 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
"""Main entry point for the multimodal processor benchmark."""
|
||||
|
||||
print("Starting multimodal processor benchmark...")
|
||||
result = benchmark_multimodal_processor(args)
|
||||
|
||||
|
||||
@@ -29,8 +29,7 @@ def generate_timeline_plot(
|
||||
itl_thresholds: list[float] | None = None,
|
||||
labels: list[str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Generate an HTML timeline plot from benchmark results.
|
||||
"""Generate an HTML timeline plot from benchmark results.
|
||||
|
||||
Args:
|
||||
results: List of per-request result dictionaries containing:
|
||||
@@ -44,8 +43,8 @@ def generate_timeline_plot(
|
||||
colors: List of colors for ITL categories (default: green, orange, red, black)
|
||||
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
|
||||
labels: Labels for ITL categories (default based on thresholds)
|
||||
"""
|
||||
|
||||
"""
|
||||
# Set defaults
|
||||
if colors is None:
|
||||
colors = ["#109618", "#FF7F0E", "#D62728"]
|
||||
@@ -118,8 +117,7 @@ def construct_timeline_data(
|
||||
itl_thresholds: list[float],
|
||||
labels: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Construct timeline data from request results.
|
||||
"""Construct timeline data from request results.
|
||||
|
||||
Args:
|
||||
requests_data: List of per-request result dictionaries
|
||||
@@ -128,6 +126,7 @@ def construct_timeline_data(
|
||||
|
||||
Returns:
|
||||
List of timeline segments for plotting
|
||||
|
||||
"""
|
||||
|
||||
def tostr(sec_time: float) -> str:
|
||||
@@ -229,8 +228,7 @@ def generate_dataset_stats_plot(
|
||||
results: list[dict[str, Any]],
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Generate a matplotlib figure with dataset statistics.
|
||||
"""Generate a matplotlib figure with dataset statistics.
|
||||
|
||||
Creates a figure with 4 subplots:
|
||||
- Top-left: Prompt tokens distribution (histogram)
|
||||
@@ -243,6 +241,7 @@ def generate_dataset_stats_plot(
|
||||
- prompt_len: Number of prompt tokens
|
||||
- output_tokens: Number of output tokens
|
||||
output_path: Path where the figure will be saved
|
||||
|
||||
"""
|
||||
# Extract data
|
||||
prompt_tokens = []
|
||||
|
||||
@@ -407,13 +407,15 @@ async def get_request(
|
||||
ramp_up_end_rps: int | None = None,
|
||||
self_timed: bool = False,
|
||||
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
|
||||
"""
|
||||
Asynchronously generates requests at a specified rate
|
||||
"""Asynchronously generates requests at a specified rate
|
||||
with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
|
||||
|
||||
Args:
|
||||
input_requests:
|
||||
A list of input requests, each represented as a SampleRequest.
|
||||
self_timed:
|
||||
If True, the requests carry their own arrival timing and no
|
||||
request rate, burstiness or ramp-up is applied.
|
||||
request_rate:
|
||||
The rate at which requests are generated (requests/s).
|
||||
burstiness (optional):
|
||||
@@ -431,6 +433,7 @@ async def get_request(
|
||||
The starting request rate for ramp-up.
|
||||
ramp_up_end_rps (optional):
|
||||
The ending request rate for ramp-up.
|
||||
|
||||
"""
|
||||
assert burstiness > 0, (
|
||||
f"A positive burstiness factor is expected, but given {burstiness}."
|
||||
@@ -526,6 +529,7 @@ def calculate_metrics_for_embeddings(
|
||||
|
||||
Returns:
|
||||
The calculated benchmark metrics.
|
||||
|
||||
"""
|
||||
total_input = 0
|
||||
total_input_sequences = 0
|
||||
@@ -585,6 +589,7 @@ def calculate_metrics(
|
||||
|
||||
Returns:
|
||||
A tuple of the benchmark metrics and the actual output lengths.
|
||||
|
||||
"""
|
||||
actual_output_lens: list[int] = []
|
||||
total_input = 0
|
||||
@@ -1571,6 +1576,7 @@ def compute_result_filename(
|
||||
|
||||
Returns:
|
||||
The computed filename path or None if no result saving is requested
|
||||
|
||||
"""
|
||||
if not (args.plot_timeline or args.save_result or args.append_result):
|
||||
return None
|
||||
|
||||
@@ -107,8 +107,7 @@ def _metric_to_json(m: MetricStats) -> dict[str, Any]:
|
||||
|
||||
@contextmanager
|
||||
def cold_startup():
|
||||
"""
|
||||
Context manager to measure cold startup time:
|
||||
"""Context manager to measure cold startup time:
|
||||
1. Uses a temporary directory for vLLM cache to avoid any pollution
|
||||
between cold startup iterations.
|
||||
2. Uses inductor's fresh_cache to clear torch.compile caches.
|
||||
@@ -132,8 +131,7 @@ def cold_startup():
|
||||
|
||||
|
||||
def run_startup_in_subprocess(engine_args, result_queue):
|
||||
"""
|
||||
Run LLM startup in a subprocess and return timing metrics via a queue.
|
||||
"""Run LLM startup in a subprocess and return timing metrics via a queue.
|
||||
This ensures complete isolation between iterations.
|
||||
"""
|
||||
try:
|
||||
@@ -229,11 +227,9 @@ def main(args: argparse.Namespace):
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
|
||||
def create_llm_and_measure_startup():
|
||||
"""
|
||||
Create LLM instance in a subprocess and measure startup time.
|
||||
"""Create LLM instance in a subprocess and measure startup time.
|
||||
Returns timing metrics, using subprocess for complete isolation.
|
||||
"""
|
||||
|
||||
# Create a queue for inter-process communication
|
||||
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
|
||||
process = multiprocessing.Process(
|
||||
|
||||
@@ -18,14 +18,14 @@ class ParameterSweep(list["ParameterSweepItem"]):
|
||||
|
||||
@classmethod
|
||||
def read_from_dict(cls, data: dict[str, dict[str, object]]):
|
||||
"""
|
||||
Read parameter sweep from a dict format where keys are names.
|
||||
"""Read parameter sweep from a dict format where keys are names.
|
||||
|
||||
Example:
|
||||
{
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9}
|
||||
}
|
||||
|
||||
"""
|
||||
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
|
||||
return cls.from_records(records)
|
||||
@@ -66,8 +66,7 @@ class ParameterSweepItem(dict[str, object]):
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""
|
||||
Get the name for this parameter sweep item.
|
||||
"""Get the name for this parameter sweep item.
|
||||
|
||||
Returns the '_benchmark_name' field if present, otherwise returns a text
|
||||
representation of all parameters.
|
||||
@@ -103,8 +102,7 @@ class ParameterSweepItem(dict[str, object]):
|
||||
return any(k in self for k in self._iter_param_key_candidates(param_key))
|
||||
|
||||
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
|
||||
"""
|
||||
Normalize a key-value pair into command-line arguments.
|
||||
"""Normalize a key-value pair into command-line arguments.
|
||||
|
||||
Returns a list containing either:
|
||||
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
|
||||
|
||||
@@ -178,8 +178,7 @@ def _json_load_bytes(path: Path) -> list[dict[str, object]]:
|
||||
|
||||
|
||||
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
"""
|
||||
Convert string values "inf", "-inf", and "nan" to their float equivalents.
|
||||
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
|
||||
|
||||
This handles the case where JSON serialization represents inf/nan as strings.
|
||||
"""
|
||||
|
||||
@@ -195,8 +195,7 @@ def run_vllm_chat(
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput]]:
|
||||
"""
|
||||
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
|
||||
"""Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
|
||||
multimodal models as it properly handles multimodal inputs and chat
|
||||
formatting. For non-multimodal models, use run_vllm() instead.
|
||||
"""
|
||||
@@ -536,6 +535,7 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
|
||||
|
||||
Returns:
|
||||
A namespace satisfying get_samples's attribute reads.
|
||||
|
||||
"""
|
||||
d = vars(args).copy()
|
||||
# random_*: prefer --random-* over legacy --input/output/prefix-len.
|
||||
@@ -627,10 +627,7 @@ def assign_loras(requests, args):
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""
|
||||
Validate command-line arguments.
|
||||
"""
|
||||
|
||||
"""Validate command-line arguments."""
|
||||
# === Deprecation and Defaulting ===
|
||||
if args.dataset is not None:
|
||||
warnings.warn(
|
||||
|
||||
@@ -73,6 +73,7 @@ def make_copy_and_call(
|
||||
|
||||
Returns:
|
||||
A wrapper function that copies inputs and calls the compiled function
|
||||
|
||||
"""
|
||||
|
||||
def copy_and_call(*args: Any) -> Any:
|
||||
@@ -122,8 +123,7 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
|
||||
|
||||
|
||||
class CompilerManager:
|
||||
"""
|
||||
A manager to manage the compilation process, including
|
||||
"""A manager to manage the compilation process, including
|
||||
caching the compiled graph, loading the compiled graph,
|
||||
and compiling the graph.
|
||||
|
||||
@@ -150,7 +150,8 @@ class CompilerManager:
|
||||
def compile_context(self, compile_range: Range) -> Generator[None, None, None]:
|
||||
"""Provide compilation context for the duration of compilation to set
|
||||
any torch global properties we want to scope to a single Inductor
|
||||
compilation (e.g. partition rules, pass context)."""
|
||||
compilation (e.g. partition rules, pass context).
|
||||
"""
|
||||
with pass_context(compile_range):
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
with inductor_partition_rule_context(
|
||||
@@ -163,8 +164,7 @@ class CompilerManager:
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the cache directory for the compiler.
|
||||
"""Initialize the cache directory for the compiler.
|
||||
|
||||
The organization of the cache directory is as follows:
|
||||
cache_dir=/path/to/hash_str/rank_i_j/prefix/
|
||||
@@ -177,7 +177,6 @@ class CompilerManager:
|
||||
base cache dir of /path/to/hash_str/rank_i_j/ ,
|
||||
to store some common compilation artifacts.
|
||||
"""
|
||||
|
||||
self.disable_cache = disable_cache
|
||||
self.cache_dir = cache_dir
|
||||
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
|
||||
@@ -438,12 +437,10 @@ def _merge_empty_only_subgraphs(
|
||||
node_to_subgraph_id: dict[fx.Node, int],
|
||||
split_op_graphs: list[int],
|
||||
) -> None:
|
||||
"""
|
||||
Merge a partition that only contains an empty allocation op into the
|
||||
"""Merge a partition that only contains an empty allocation op into the
|
||||
previous partition. This avoids generating standalone empty submodules,
|
||||
which can lead to empty cudagraph captures.
|
||||
"""
|
||||
|
||||
nodes_by_subgraph_id: dict[int, list[fx.Node]] = defaultdict(list)
|
||||
for node, subgraph_id in node_to_subgraph_id.items():
|
||||
nodes_by_subgraph_id[subgraph_id].append(node)
|
||||
@@ -637,8 +634,7 @@ def wrap_with_cudagraph_if_needed(
|
||||
is_first_graph: bool,
|
||||
is_last_graph: bool,
|
||||
) -> Any:
|
||||
"""
|
||||
Wrap a piecewise backend with CUDA graph wrapper if needed.
|
||||
"""Wrap a piecewise backend with CUDA graph wrapper if needed.
|
||||
This function is shared between VllmBackend and
|
||||
construct_serializable_fn_from_inductor_cache.
|
||||
|
||||
@@ -651,6 +647,7 @@ def wrap_with_cudagraph_if_needed(
|
||||
|
||||
Returns:
|
||||
The wrapped backend if CUDA graphs are enabled, otherwise the original backend
|
||||
|
||||
"""
|
||||
if (
|
||||
not compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
|
||||
@@ -883,8 +880,8 @@ class VllmBackend:
|
||||
sym_shape_indices
|
||||
- returns_tuple_map: dict mapping submod_name to
|
||||
returns_tuple
|
||||
"""
|
||||
|
||||
"""
|
||||
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
|
||||
return None, None, None
|
||||
|
||||
|
||||
@@ -8,8 +8,7 @@ from vllm.config import CUDAGraphMode, VllmConfig
|
||||
|
||||
|
||||
class AbstractStaticGraphWrapper(Protocol):
|
||||
"""
|
||||
StaticGraphWrapper interface that allows platforms to wrap a callable
|
||||
"""StaticGraphWrapper interface that allows platforms to wrap a callable
|
||||
to be captured as a static graph.
|
||||
"""
|
||||
|
||||
@@ -20,8 +19,7 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
runtime_mode: CUDAGraphMode,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Initializes the StaticGraphWrapper class with graph capturing and
|
||||
"""Initializes the StaticGraphWrapper class with graph capturing and
|
||||
execution-related configurations.
|
||||
|
||||
Args:
|
||||
@@ -31,15 +29,16 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
graph runtime. See CUDAGraphMode in vllm/config.py.
|
||||
Note that only the subset enum `NONE`, `PIECEWISE` and `FULL`
|
||||
are used as concrete runtime mode for cudagraph dispatching.
|
||||
|
||||
Keyword Args:
|
||||
kwargs: Additional keyword arguments for platform-specific
|
||||
configurations.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
"""
|
||||
Executes the wrapped callable.
|
||||
"""Executes the wrapped callable.
|
||||
|
||||
If the current runtime mode in the ForwardContext matches the runtime
|
||||
mode of this instance, it replays the CUDAGraph or captures it using
|
||||
@@ -53,5 +52,6 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
|
||||
Returns:
|
||||
Any: Output of the executed callable.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -172,8 +172,7 @@ def patch_pytree_map_over_slice():
|
||||
|
||||
|
||||
class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
|
||||
"""
|
||||
A wrapper around a compiled function by vllm. It will forward the tensor
|
||||
"""A wrapper around a compiled function by vllm. It will forward the tensor
|
||||
inputs to the compiled function and return the result.
|
||||
It also implements a serialization interface to support PyTorch's precompile
|
||||
with custom backend, so that we can save and load the compiled function on
|
||||
@@ -410,9 +409,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
|
||||
|
||||
@property
|
||||
def co_name(self) -> Literal["VllmSerializableFunction"]:
|
||||
"""
|
||||
Used for depyf debugging.
|
||||
"""
|
||||
"""Used for depyf debugging."""
|
||||
return "VllmSerializableFunction"
|
||||
|
||||
|
||||
@@ -445,6 +442,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
|
||||
If modifying the backend creation/wrapping logic, consider updating both.
|
||||
|
||||
Args:
|
||||
fake_mode: FakeTensorMode used to unpickle the cached graph.
|
||||
state: Deserialized state dict containing graph_module, example_inputs,
|
||||
prefix, sym_tensor_indices, is_encoder, etc.
|
||||
standalone_compile_artifacts: The StandaloneCompiledArtifacts containing
|
||||
@@ -455,6 +453,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
|
||||
|
||||
Returns:
|
||||
A VllmSerializableFunction that can be called directly.
|
||||
|
||||
"""
|
||||
from vllm.compilation.backends import (
|
||||
VllmBackend,
|
||||
|
||||
@@ -154,6 +154,7 @@ def generate_execution_code(
|
||||
consts is a list of non-primitive constant objects referenced
|
||||
by the generated code via __vllm_consts__. These objects are
|
||||
kept alive for the lifetime of the compiled function.
|
||||
|
||||
"""
|
||||
code, submod_names, consts = generate_execution_code_with_name(
|
||||
split_gm, "execution_fn", with_submod=True
|
||||
@@ -181,6 +182,7 @@ def compile_execution_fn(
|
||||
|
||||
Returns:
|
||||
A callable that executes the stitching logic.
|
||||
|
||||
"""
|
||||
trace_structured(
|
||||
"artifact",
|
||||
|
||||
@@ -25,9 +25,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompilerInterface:
|
||||
"""
|
||||
The interface for a compiler that can be used by vLLM.
|
||||
"""
|
||||
"""The interface for a compiler that can be used by vLLM."""
|
||||
|
||||
# The name of the compiler, e.g. inductor.
|
||||
# This is a class-level attribute.
|
||||
@@ -36,8 +34,7 @@ class CompilerInterface:
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
) -> None:
|
||||
"""
|
||||
when the vLLM process uses `cache_dir` as the cache directory,
|
||||
"""When the vLLM process uses `cache_dir` as the cache directory,
|
||||
the compiler should initialize itself with the cache directory,
|
||||
e.g. by re-directing its own cache directory to a sub-directory.
|
||||
|
||||
@@ -52,8 +49,7 @@ class CompilerInterface:
|
||||
pass
|
||||
|
||||
def compute_hash(self, vllm_config: VllmConfig) -> str:
|
||||
"""
|
||||
Gather all the relevant information from the vLLM config,
|
||||
"""Gather all the relevant information from the vLLM config,
|
||||
to compute a hash so that we can cache the compiled model.
|
||||
|
||||
See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash]
|
||||
@@ -71,8 +67,7 @@ class CompilerInterface:
|
||||
compile_range: Range,
|
||||
key: str | None = None,
|
||||
) -> tuple[Callable[..., Any] | None, Any | None]:
|
||||
"""
|
||||
Compile the graph with the given example inputs and compiler config,
|
||||
"""Compile the graph with the given example inputs and compiler config,
|
||||
with a range. The `compile_range` specifies the range of the inputs,
|
||||
it could be concrete size (if compile_sizes is provided), e.g. [4, 4]
|
||||
or a range [5, 8].
|
||||
@@ -105,8 +100,7 @@ class CompilerInterface:
|
||||
graph_index: int,
|
||||
compile_range: Range,
|
||||
) -> Callable[..., Any]:
|
||||
"""
|
||||
Load the compiled function from the handle.
|
||||
"""Load the compiled function from the handle.
|
||||
Raises an error if the handle is invalid.
|
||||
|
||||
The handle is the second return value of the `compile` function.
|
||||
@@ -115,8 +109,7 @@ class CompilerInterface:
|
||||
|
||||
|
||||
class AlwaysHitShapeEnv:
|
||||
"""
|
||||
Why do we need this class:
|
||||
"""Why do we need this class:
|
||||
|
||||
For normal `torch.compile` usage, every compilation will have
|
||||
one Dynamo bytecode compilation and one Inductor compilation.
|
||||
@@ -160,7 +153,8 @@ def _get_vllm_functorch_config() -> dict[str, Any]:
|
||||
"""Return the functorch config overrides that vLLM applies at compile time.
|
||||
|
||||
Used by both set_functorch_config() and get_inductor_factors() to ensure
|
||||
the compile-time config and cache key are always consistent."""
|
||||
the compile-time config and cache key are always consistent.
|
||||
"""
|
||||
cfg: dict[str, Any] = {}
|
||||
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
|
||||
cfg["bundled_autograd_cache"] = False
|
||||
@@ -249,8 +243,7 @@ def _patch_standalone_compile_atomic_save() -> None:
|
||||
|
||||
|
||||
class InductorStandaloneAdaptor(CompilerInterface):
|
||||
"""
|
||||
The adaptor for the Inductor compiler.
|
||||
"""The adaptor for the Inductor compiler.
|
||||
Requires PyTorch 2.8+.
|
||||
This is not on by default yet, but we plan to turn it on by default for
|
||||
PyTorch 2.8.
|
||||
@@ -447,9 +440,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
|
||||
|
||||
|
||||
class InductorAdaptor(CompilerInterface):
|
||||
"""
|
||||
The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7.
|
||||
"""
|
||||
"""The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7."""
|
||||
|
||||
name = "inductor"
|
||||
|
||||
@@ -727,8 +718,7 @@ class InductorAdaptor(CompilerInterface):
|
||||
return compiled_graph
|
||||
|
||||
def metrics_context(self) -> contextlib.AbstractContextManager[Any]:
|
||||
"""
|
||||
This method returns the Dynamo metrics context (if it exists,
|
||||
"""This method returns the Dynamo metrics context (if it exists,
|
||||
otherwise a null context). It is used by various compile components.
|
||||
Present in torch>=2.6, it's used inside FxGraphCache in
|
||||
torch==2.6 (but not after). It might also be used in various other
|
||||
|
||||
@@ -38,7 +38,7 @@ class CUDAGraphStat:
|
||||
|
||||
|
||||
class CUDAGraphLogging:
|
||||
"""Aggregate and log cudagraph metrics"""
|
||||
"""Aggregate and log cudagraph metrics."""
|
||||
|
||||
COLUMN_HEADERS = [
|
||||
"Unpadded Tokens",
|
||||
|
||||
@@ -56,8 +56,7 @@ def should_torch_compile_mm_encoder(vllm_config: VllmConfig) -> bool:
|
||||
|
||||
|
||||
def ignore_torch_compile(cls: type[_T]) -> type[_T]:
|
||||
"""
|
||||
A decorator to ignore support_torch_compile decorator
|
||||
"""A decorator to ignore support_torch_compile decorator
|
||||
on the class. This is useful when a parent class has
|
||||
a support_torch_compile decorator, but we don't want to
|
||||
compile the class `cls` that inherits the parent class.
|
||||
@@ -76,9 +75,7 @@ def ignore_torch_compile(cls: type[_T]) -> type[_T]:
|
||||
|
||||
|
||||
def _should_ignore_torch_compile(cls: type[_T]) -> bool:
|
||||
"""
|
||||
Check if the class should be ignored for torch.compile.
|
||||
"""
|
||||
"""Check if the class should be ignored for torch.compile."""
|
||||
return getattr(cls, IGNORE_COMPILE_KEY, False)
|
||||
|
||||
|
||||
@@ -125,8 +122,7 @@ def support_torch_compile(
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
) -> Callable[[type[_T]], type[_T]] | type[_T]:
|
||||
"""
|
||||
A decorator to add support for compiling the forward method of a class.
|
||||
"""A decorator to add support for compiling the forward method of a class.
|
||||
|
||||
Usage 1: use directly as a decorator without arguments:
|
||||
|
||||
@@ -338,7 +334,6 @@ def _support_torch_compile(
|
||||
is_encoder: bool = False,
|
||||
) -> type[_T]:
|
||||
"""Internal implementation of support_torch_compile decorator."""
|
||||
|
||||
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
|
||||
# support decorating multiple times
|
||||
return cls
|
||||
@@ -736,8 +731,7 @@ def _support_torch_compile(
|
||||
def maybe_use_cudagraph_partition_wrapper(
|
||||
vllm_config: VllmConfig,
|
||||
) -> Generator[None, None, None]:
|
||||
"""
|
||||
Context manager to set/unset customized cudagraph partition wrappers.
|
||||
"""Context manager to set/unset customized cudagraph partition wrappers.
|
||||
|
||||
If we're using Inductor-based graph partitioning, we currently have the
|
||||
whole `fx.Graph` before Inductor lowering and the piecewise
|
||||
|
||||
@@ -12,12 +12,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def should_split(node: torch.fx.Node, splitting_ops: list[str]) -> bool:
|
||||
"""
|
||||
Check if a node should be split for dynamo graph partition.
|
||||
"""Check if a node should be split for dynamo graph partition.
|
||||
It operates on dynamo graph, so the node.target can be anything.
|
||||
We need to check and split only on OpOverload and OpOverloadPacket.
|
||||
"""
|
||||
|
||||
if node.op != "call_function":
|
||||
return False
|
||||
|
||||
@@ -50,6 +48,7 @@ def inductor_partition_rule_context(
|
||||
|
||||
Args:
|
||||
splitting_ops: List of operator names to partition on.
|
||||
|
||||
"""
|
||||
if not splitting_ops:
|
||||
logger.debug("No partition ops provided; skipping rule registration.")
|
||||
|
||||
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
|
||||
|
||||
|
||||
class ActivationQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
Base class for Activation+Quant fusions.
|
||||
"""Base class for Activation+Quant fusions.
|
||||
Should not be used directly.
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Fp8StaticQuant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kFp8StaticTensorSym)
|
||||
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Nvfp4Quant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kNvfp4Dynamic)
|
||||
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
Supports group_size 128 and 64 via QuantKey.
|
||||
Parameterized on is_scale_transposed for different scale layouts.
|
||||
"""
|
||||
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
|
||||
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
|
||||
|
||||
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
|
||||
"""
|
||||
for node in match.nodes:
|
||||
if node.target == _IR_RMS_NORM_OP:
|
||||
x, weight = node.args[0], node.args[1]
|
||||
@@ -347,8 +348,7 @@ class BasePattern:
|
||||
|
||||
|
||||
class AllReduceRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm before attn in the first Transformer block.
|
||||
"""
|
||||
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
|
||||
"""
|
||||
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
|
||||
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
|
||||
|
||||
|
||||
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for Attention+Fp8StaticQuant.
|
||||
"""Fusion for Attention+Fp8StaticQuant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class AttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for Attention+Nvfp4Quant.
|
||||
"""Fusion for Attention+Nvfp4Quant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
|
||||
|
||||
|
||||
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto attention if supported.
|
||||
"""This pass fuses post-attention quantization onto attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
|
||||
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
"""Utility for inputs to the pattern"""
|
||||
"""Utility for inputs to the pattern."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8StaticQuant.
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Nvfp4Quant.
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -34,8 +34,7 @@ P = ParamSpec("P")
|
||||
|
||||
|
||||
class QkNormRopePattern:
|
||||
"""
|
||||
Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
"""Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
|
||||
Unfused (conceptually):
|
||||
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
|
||||
|
||||
@@ -106,8 +106,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class QkNormRopeKvCachePattern:
|
||||
"""
|
||||
Match the unfused sequence:
|
||||
"""Match the unfused sequence:
|
||||
q, k, v = split(qkv, ...)
|
||||
q = rms_norm(q.view(heads), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads), k_weight).view(flat)
|
||||
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
|
||||
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
|
||||
|
||||
class FusedRMSQuantKey(NamedTuple):
|
||||
"""
|
||||
Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
"""Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
quant: type of quantization
|
||||
fused_add: does the op also perform the residual add
|
||||
"""
|
||||
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
|
||||
|
||||
|
||||
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
|
||||
|
||||
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
"""This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
ops into an aiter rms_norm_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
into a aiter rms_norm_with_add_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
|
||||
independent fused ``rms_norm_group_fp8_quant`` ops.
|
||||
|
||||
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
|
||||
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
|
||||
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
|
||||
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
with rocm_aiter_fused_rms_gated_fp8_group_quant.
|
||||
|
||||
The norm operates per-head on (N*H, D) tensors. The compiler folds the
|
||||
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
|
||||
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
ops into an aiter silu_and_mul_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
|
||||
|
||||
class AddAiterRMSNormPadPattern:
|
||||
"""
|
||||
This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
with a custom triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
|
||||
|
||||
|
||||
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
with an triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
class MLADualRMSNormPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
AITER's ``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
|
||||
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
],
|
||||
]
|
||||
):
|
||||
"""
|
||||
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
|
||||
|
||||
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
|
||||
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
|
||||
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
is_neox: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This impl fetches the KV cache and slot mapping from the forward context,
|
||||
"""This impl fetches the KV cache and slot mapping from the forward context,
|
||||
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
|
||||
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
|
||||
that is passed to unified_attention to signal a side effect and
|
||||
@@ -98,8 +97,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class RopeStaticQQuantKVCachePattern:
|
||||
"""
|
||||
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
dependency ordering.
|
||||
"""
|
||||
|
||||
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
|
||||
|
||||
|
||||
class RopeReshapeKVCachePattern:
|
||||
"""
|
||||
This pattern matches the following unfused inplace ops:
|
||||
"""This pattern matches the following unfused inplace ops:
|
||||
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
|
||||
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
|
||||
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
|
||||
|
||||
|
||||
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses the rotary embedding and KV cache update operations
|
||||
"""This pass fuses the rotary embedding and KV cache update operations
|
||||
into a single fused kernel if available.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
|
||||
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
|
||||
tp_size: int,
|
||||
element_size: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Calculate the minimum token threshold for applying sequence parallelism.
|
||||
"""Calculate the minimum token threshold for applying sequence parallelism.
|
||||
|
||||
Returns None if sequence parallelism should not be applied based on model size.
|
||||
|
||||
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
|
||||
|
||||
class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass enables sequence parallelism for models.
|
||||
"""This pass enables sequence parallelism for models.
|
||||
It identifies patterns where an AllReduce operation is followed by
|
||||
an RMSNorm (or RMSNorm and then Quantization) operation.
|
||||
These patterns are replaced with a ReduceScatter operation, followed by
|
||||
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
"""
|
||||
Determines if sequence parallelism should be applied for the given
|
||||
"""Determines if sequence parallelism should be applied for the given
|
||||
compile range.
|
||||
|
||||
SP is only beneficial for larger batch sizes where the communication
|
||||
|
||||
@@ -64,14 +64,12 @@ def _hash_source_cached(*srcs: str | type | types.FunctionType) -> str:
|
||||
|
||||
|
||||
class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
A custom graph pass that uses a hash of its source as the UUID.
|
||||
"""A custom graph pass that uses a hash of its source as the UUID.
|
||||
This is defined as a convenience and should work in most cases.
|
||||
"""
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
"""Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
This should depend on the pass implementation, so that changes to the
|
||||
pass result in recompilation.
|
||||
By default, the object source is hashed.
|
||||
@@ -80,14 +78,14 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_source(*srcs: str | Any) -> str:
|
||||
"""
|
||||
Utility method to hash the sources of functions or objects.
|
||||
"""Utility method to hash the sources of functions or objects.
|
||||
|
||||
Args:
|
||||
srcs: strings or objects to add to the hash.
|
||||
Objects and functions have their source inspected.
|
||||
Results are cached by resolved types to avoid repeated
|
||||
inspect.getsource() calls.
|
||||
|
||||
"""
|
||||
# Resolve instances to their class for a hashable cache key.
|
||||
cache_key = tuple(
|
||||
@@ -98,11 +96,11 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_dict(dict_: dict[Any, Any]) -> str:
|
||||
"""
|
||||
Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
"""Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
|
||||
Returns:
|
||||
A sha256 hash of the json rep of the dictionary.
|
||||
|
||||
"""
|
||||
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
@@ -112,8 +110,7 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
|
||||
class CallableInductorPass(InductorPass):
|
||||
"""
|
||||
This class is a wrapper for a callable that automatically provides an
|
||||
"""This class is a wrapper for a callable that automatically provides an
|
||||
implementation of the UUID.
|
||||
"""
|
||||
|
||||
@@ -131,8 +128,7 @@ class CallableInductorPass(InductorPass):
|
||||
|
||||
|
||||
def enable_fake_mode(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
"""Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
create or run things with real tensors.
|
||||
"""
|
||||
|
||||
|
||||
@@ -70,8 +70,7 @@ def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
|
||||
|
||||
|
||||
class UnsafeCloneEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
"""This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
|
||||
contents of non-donated graph inputs.
|
||||
|
||||
|
||||
@@ -19,8 +19,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
"""This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
The maybe_inplace overloads have the same signature as the default overload
|
||||
so the pass simply replaces the called overload.
|
||||
That makes the graph properly functional.
|
||||
|
||||
@@ -23,9 +23,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRLoweringPass(VllmInductorPass):
|
||||
"""
|
||||
This pass lowers vLLM IR ops to their implementations the priority list.
|
||||
"""
|
||||
"""This pass lowers vLLM IR ops to their implementations the priority list."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
super().__init__(vllm_config)
|
||||
@@ -113,8 +111,7 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
IR op priority & impl sources affect lowering pass output,
|
||||
"""IR op priority & impl sources affect lowering pass output,
|
||||
so we include them in the cache key.
|
||||
"""
|
||||
priorities = {name: op.get_priority() for name, op in IrOp.registry.items()}
|
||||
|
||||
@@ -71,8 +71,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Function decorator that turns on inductor pattern match debug
|
||||
"""Function decorator that turns on inductor pattern match debug
|
||||
for the duration of the call.
|
||||
Used to avoid logging builtin Inductor pattern matching.
|
||||
"""
|
||||
@@ -89,8 +88,7 @@ def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
|
||||
|
||||
class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
The pass manager for post-grad passes.
|
||||
"""The pass manager for post-grad passes.
|
||||
It handles configuration, adding custom passes, and running passes.
|
||||
It supports uuid for the Inductor code cache. That includes torch<2.6
|
||||
support using pickling (in .inductor_pass.CustomGraphPass).
|
||||
@@ -236,8 +234,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes.append(pass_)
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
"""The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
affects compilation caching. Its uuid depends on the UUIDs of all
|
||||
dependent passes and the pass config. See InductorPass for more info.
|
||||
"""
|
||||
|
||||
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FixFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
"""This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
After this pass, DCE (dead-code elimination) should never be run,
|
||||
as de-functionalized nodes may appear as dead code.
|
||||
|
||||
@@ -260,9 +259,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self.nodes_to_remove.clear()
|
||||
|
||||
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]) -> None:
|
||||
"""
|
||||
Stage a node (or nodes) for removal at the end of the pass.
|
||||
"""
|
||||
"""Stage a node (or nodes) for removal at the end of the pass."""
|
||||
if isinstance(node_or_nodes, torch.fx.Node):
|
||||
self.nodes_to_remove.append(node_or_nodes)
|
||||
else:
|
||||
@@ -275,8 +272,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
mutated_args: dict[int, torch.fx.Node | str],
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
De-functionalize a node by replacing it with a call to the original.
|
||||
"""De-functionalize a node by replacing it with a call to the original.
|
||||
It also replaces the getitem users with the mutated arguments.
|
||||
See replace_users_with_mutated_args and insert_defunctionalized.
|
||||
"""
|
||||
@@ -287,14 +283,14 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
def replace_users_with_mutated_args(
|
||||
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
|
||||
) -> None:
|
||||
"""
|
||||
Replace mutated getitem users of the auto-functionalized node with the
|
||||
"""Replace mutated getitem users of the auto-functionalized node with the
|
||||
mutated arguments.
|
||||
|
||||
Args:
|
||||
node: The auto-functionalized node
|
||||
mutated_args: The mutated arguments, indexed by getitem index.
|
||||
If the value of an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
"""
|
||||
for idx, user in self.getitem_users(node).items():
|
||||
# Some functionalized nodes may return both a result at getitem[0]
|
||||
@@ -310,8 +306,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self._remove(user)
|
||||
|
||||
def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]:
|
||||
"""
|
||||
Returns the operator.getitem users of the auto-functionalized node,
|
||||
"""Returns the operator.getitem users of the auto-functionalized node,
|
||||
indexed by the index they are getting.
|
||||
"""
|
||||
users = {}
|
||||
@@ -327,8 +322,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: torch.fx.Node,
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Insert a new defunctionalized node into the graph before node.
|
||||
"""Insert a new defunctionalized node into the graph before node.
|
||||
If one of the kwargs is 'out', provide args directly,
|
||||
as node.kwargs cannot be used.
|
||||
See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
|
||||
@@ -338,6 +332,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: The auto-functionalized node to defunctionalize
|
||||
args: If we cannot use kwargs, specify args directly.
|
||||
If an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
""" # noqa: E501
|
||||
assert is_func(node, auto_functionalized), (
|
||||
f"node must be auto-functionalized, is {node} instead"
|
||||
|
||||
@@ -16,8 +16,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NoOpEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This is an inductor pass that removes redundant reshape/slice operations.
|
||||
"""This is an inductor pass that removes redundant reshape/slice operations.
|
||||
It is required for RMSNorm-quant fusion to work properly.
|
||||
That's because apply_fp8_linear adds a reshape, which is redundant
|
||||
in the 2D-case. Additionally, torch internal no-op elimination pass does
|
||||
@@ -106,8 +105,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
|
||||
# ---------------------- Shape comparison helpers ----------------------
|
||||
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
|
||||
"""
|
||||
This function checks if two dimensions are equivalent.
|
||||
"""This function checks if two dimensions are equivalent.
|
||||
|
||||
Args:
|
||||
dim: The dimension arg to reshape/slice
|
||||
@@ -119,6 +117,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
There are two cases in which the dimensions are equivalent:
|
||||
1. The dimensions are equal (both integers)
|
||||
2. The dimensions both correspond to the same SymInt
|
||||
|
||||
"""
|
||||
# Case 1
|
||||
return statically_known_true(dim == i_dim) # type: ignore[no-any-return]
|
||||
|
||||
@@ -6,8 +6,7 @@ from ..vllm_inductor_pass import VllmInductorPass
|
||||
|
||||
|
||||
class PostCleanupPass(VllmInductorPass):
|
||||
"""
|
||||
This pass performs cleanup after custom passes.
|
||||
"""This pass performs cleanup after custom passes.
|
||||
It topologically sorts the graph and removes unused nodes.
|
||||
This is needed because the pattern matcher does not guarantee producing
|
||||
a topologically sorted graph, and there may be unused nodes left around.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
"""Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
assignment if there are no users for the inplace tensor written to by
|
||||
the slice_scatter call.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
input tensor with the same split sizes.
|
||||
|
||||
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
|
||||
@@ -10,9 +9,10 @@ that CSE fails to merge. This pass detects and replaces the duplicates
|
||||
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
|
||||
see a single split node with all users attached.
|
||||
|
||||
See also:
|
||||
See Also:
|
||||
- vLLM #33295 (original issue)
|
||||
- PyTorch #174472 (upstream CSE gap)
|
||||
|
||||
"""
|
||||
|
||||
import operator
|
||||
@@ -30,7 +30,8 @@ logger = init_logger(__name__)
|
||||
|
||||
class SplitCoalescingPass(VllmInductorPass):
|
||||
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
|
||||
node when they share the same input tensor and split sizes."""
|
||||
node when they share the same input tensor and split sizes.
|
||||
"""
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
|
||||
@@ -35,8 +35,7 @@ class InductorCompilationConfig:
|
||||
|
||||
|
||||
class VllmInductorPass(InductorPass):
|
||||
"""
|
||||
An inductor pass with access to vLLM PassConfig.
|
||||
"""An inductor pass with access to vLLM PassConfig.
|
||||
It provides timing, logging, and dumping utilities.
|
||||
"""
|
||||
|
||||
@@ -93,8 +92,7 @@ def get_match_table() -> dict[str, int]:
|
||||
|
||||
|
||||
class VllmPatternMatcherPass(VllmInductorPass):
|
||||
"""
|
||||
A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
"""A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
Provides pattern registration with match counting, debug dumping, and logging.
|
||||
"""
|
||||
|
||||
@@ -109,7 +107,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
)
|
||||
|
||||
def _replace_op_overloads(self, string: str) -> str:
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations"""
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations."""
|
||||
return str(
|
||||
self._OP_OVERLOAD_PATTERN.sub(
|
||||
lambda m: f"torch.ops.{m.group(1)}.{m.group(2)}",
|
||||
@@ -123,8 +121,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
logger.debug("fusion pass matches: %s", dict(cls.match_table))
|
||||
|
||||
def dump_patterns(self, config: VllmConfig, pm_pass: PatternMatcherPass) -> None:
|
||||
"""
|
||||
If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
"""If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
into the debug_dump_path folder next to the dumped fx graphs.
|
||||
|
||||
This method does its best to print something that looks like Python code
|
||||
@@ -195,8 +192,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
"""
|
||||
A pattern/replacement pair for FX graph fusion.
|
||||
"""A pattern/replacement pair for FX graph fusion.
|
||||
|
||||
Implement the three abstract members below, then pass
|
||||
instances to VllmFusionPatternMatcherPass.register(). The pass will
|
||||
@@ -216,8 +212,7 @@ class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
@property
|
||||
@abstractmethod
|
||||
def replacement(self) -> Callable[P, R]:
|
||||
"""
|
||||
Returns a closure defining the FX subgraph to
|
||||
"""Returns a closure defining the FX subgraph to
|
||||
substitute in place of each match.
|
||||
"""
|
||||
...
|
||||
@@ -294,8 +289,7 @@ def _remove_noop_permutes(gm: fx.GraphModule) -> None:
|
||||
|
||||
|
||||
class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
"""A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
Subclasses register patterns via self.register() in their own __init__.
|
||||
"""
|
||||
|
||||
|
||||
@@ -96,8 +96,7 @@ class PiecewiseBackend:
|
||||
compiled_runnables: dict[str, Callable[..., Any]] | None = None,
|
||||
submod_name: str = "",
|
||||
):
|
||||
"""
|
||||
The backend for piecewise compilation.
|
||||
"""The backend for piecewise compilation.
|
||||
It mainly handles the compilation of static shapes and
|
||||
dispatching based on runtime shape.
|
||||
|
||||
|
||||
@@ -45,8 +45,7 @@ def _compilation_context() -> Generator[None, None, None]:
|
||||
|
||||
|
||||
class TorchCompileWithNoGuardsWrapper:
|
||||
"""
|
||||
A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
"""A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
|
||||
When guards are dropped, the first time __call__ is invoked, a single
|
||||
compilation is triggered. Dynamo should never be traced again after that
|
||||
@@ -272,8 +271,7 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
@contextmanager
|
||||
def _dispatch_to_compiled_code(self) -> Generator[None, None, None]:
|
||||
# noqa: E501
|
||||
"""
|
||||
Context manager to dispatch to internally compiled code for torch<2.8.
|
||||
"""Context manager to dispatch to internally compiled code for torch<2.8.
|
||||
Why does this work? Because Dynamo guarantees that the compiled
|
||||
bytecode has exactly the same arguments, cell variables, and free
|
||||
variables as the original code. Therefore we can directly switch
|
||||
@@ -291,9 +289,7 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
|
||||
|
||||
def reset_compile_wrapper(model: torch.nn.Module) -> None:
|
||||
"""
|
||||
Clean up compiled model and captured CUDA graphs for elastic EP.
|
||||
"""
|
||||
"""Clean up compiled model and captured CUDA graphs for elastic EP."""
|
||||
if not isinstance(model, TorchCompileWithNoGuardsWrapper) and hasattr(
|
||||
model, "model"
|
||||
):
|
||||
|
||||
@@ -150,8 +150,7 @@ class AttentionConfig:
|
||||
return self.indexer_kv_dtype
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
|
||||
@@ -248,8 +248,7 @@ class CacheConfig:
|
||||
KV offloading is only activated when kv_offloading_size is set."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+15
-24
@@ -36,7 +36,8 @@ logger = init_logger(__name__)
|
||||
|
||||
class CompilationMode(enum.IntEnum):
|
||||
"""The compilation approach used for torch.compile-based compilation of the
|
||||
model."""
|
||||
model.
|
||||
"""
|
||||
|
||||
NONE = 0
|
||||
"""No torch.compile compilation is applied, model runs in fully eager pytorch mode.
|
||||
@@ -192,12 +193,10 @@ class PassConfig:
|
||||
# TODO(luka) better pass enabling system.
|
||||
|
||||
def flashinfer_max_size(self, world_size: int) -> int | None:
|
||||
"""
|
||||
Returns the max communication size in bytes for flashinfer
|
||||
"""Returns the max communication size in bytes for flashinfer
|
||||
allreduce fusion for the given world size. Returns None if world size
|
||||
is not supported by configs as it's not supported by flashinfer.
|
||||
"""
|
||||
|
||||
MiB = 1024 * 1024
|
||||
FI_SUPPORTED_WORLD_SIZES = [2, 4, 8, 16]
|
||||
if world_size not in FI_SUPPORTED_WORLD_SIZES:
|
||||
@@ -223,12 +222,10 @@ class PassConfig:
|
||||
return FI_ALLREDUCE_FUSION_MAX_SIZE_MB.get(capability.to_int(), {})
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
"""
|
||||
|
||||
return hash_factors(get_hash_factors(self, set()))
|
||||
|
||||
@field_validator(
|
||||
@@ -317,8 +314,7 @@ class PassConfig:
|
||||
self.fuse_rope_kvcache_cat_mla = False
|
||||
|
||||
def log_enabled_passes(self) -> None:
|
||||
"""
|
||||
Log the enabled custom fusion passes.
|
||||
"""Log the enabled custom fusion passes.
|
||||
This is called at the end of VLLMConfig post_init,
|
||||
after all defaults are finalized.
|
||||
TODO also log the compile ranges for which this is enabled.
|
||||
@@ -341,7 +337,8 @@ class PassConfig:
|
||||
class DynamicShapesType(str, enum.Enum):
|
||||
"""Types of dynamic shapes handling in torch.compile().
|
||||
see Dynamic shapes and vllm guard dropping in torch_compile.md
|
||||
for more details."""
|
||||
for more details.
|
||||
"""
|
||||
|
||||
BACKED = "backed"
|
||||
"""Use backed dynamic shapes. torch.compile() guards on backed dynamic
|
||||
@@ -391,10 +388,7 @@ class DynamicShapesConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash for DynamicShapesConfig
|
||||
"""
|
||||
|
||||
"""Provide a hash for DynamicShapesConfig."""
|
||||
from vllm.config.utils import get_hash_factors, hash_factors
|
||||
|
||||
factors = get_hash_factors(self, set())
|
||||
@@ -790,8 +784,7 @@ class CompilationConfig:
|
||||
]
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
@@ -856,8 +849,7 @@ class CompilationConfig:
|
||||
@field_validator("mode", mode="before")
|
||||
@classmethod
|
||||
def validate_mode_before(cls, value: Any) -> Any:
|
||||
"""
|
||||
Enable parsing the `mode` field from string mode names.
|
||||
"""Enable parsing the `mode` field from string mode names.
|
||||
Accepts both integers (0-3) and string names, like NONE, STOCK_TORCH_COMPILE,
|
||||
DYNAMO_TRACE_ONCE, VLLM_COMPILE.
|
||||
"""
|
||||
@@ -1079,15 +1071,17 @@ class CompilationConfig:
|
||||
prefix: str = "",
|
||||
is_encoder: bool = False,
|
||||
) -> str | Callable:
|
||||
"""
|
||||
Initialize the backend for the compilation config from a vllm config.
|
||||
"""Initialize the backend for the compilation config from a vllm config.
|
||||
|
||||
Arguments:
|
||||
vllm_config: The vllm config to initialize the backend from.
|
||||
prefix: Cache directory prefix for this compiled module.
|
||||
is_encoder: Whether this module is used in an encoder (as
|
||||
opposed to a text backbone).
|
||||
|
||||
Returns:
|
||||
The backend for the compilation config.
|
||||
|
||||
"""
|
||||
if self.mode is None:
|
||||
raise ValueError(
|
||||
@@ -1122,7 +1116,6 @@ class CompilationConfig:
|
||||
configs are set. This includes:
|
||||
- initialize compile_sizes
|
||||
"""
|
||||
|
||||
computed_compile_sizes: list[int] = []
|
||||
if self.compile_sizes is not None:
|
||||
# de-duplicate the sizes provided by the config
|
||||
@@ -1321,13 +1314,11 @@ class CompilationConfig:
|
||||
return self.backend == "inductor" and self.mode != CompilationMode.NONE
|
||||
|
||||
def custom_op_log_check(self):
|
||||
"""
|
||||
This method logs the enabled/disabled custom ops and checks that the
|
||||
"""This method logs the enabled/disabled custom ops and checks that the
|
||||
passed custom_ops field only contains relevant ops.
|
||||
It is called at the end of set_current_vllm_config,
|
||||
after the custom ops have been instantiated.
|
||||
"""
|
||||
|
||||
if len(self.enabled_custom_ops) + len(self.disabled_custom_ops) == 0:
|
||||
logger.debug("No custom ops found in model.")
|
||||
return
|
||||
|
||||
@@ -28,8 +28,7 @@ class DeviceConfig:
|
||||
`__post_init__`."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -31,8 +31,7 @@ class EncoderCacheManagerConfig:
|
||||
|
||||
|
||||
class EncoderCacheManagerMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
"""Abstract Metadata used to communicate between the
|
||||
Scheduler EncoderCacheManager and Worker EncoderCacheManager.
|
||||
"""
|
||||
|
||||
|
||||
@@ -63,8 +63,7 @@ class ECTransferConfig:
|
||||
Only supported in V1."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+7
-15
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@config
|
||||
class IrOpPriorityConfig:
|
||||
"""
|
||||
Configuration for vLLM IR op priority for dispatching/lowering during the
|
||||
"""Configuration for vLLM IR op priority for dispatching/lowering during the
|
||||
forward pass. Each member is a list of strings, which will be installed
|
||||
in worker init via vllm.ir.ops.<op_name>.set_default().
|
||||
A single comma-separated string is accepted as well,
|
||||
@@ -38,8 +37,7 @@ class IrOpPriorityConfig:
|
||||
"""Priority list for vllm.ir.ops.gelu_and_mul_sparse"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
|
||||
@@ -71,8 +69,7 @@ class IrOpPriorityConfig:
|
||||
return value
|
||||
|
||||
def _iter_op_priorities(self):
|
||||
"""
|
||||
Yield (IrOp, priority_list) for each field, after importing platform
|
||||
"""Yield (IrOp, priority_list) for each field, after importing platform
|
||||
kernels and validating each entry.
|
||||
"""
|
||||
from vllm.ir.op import IrOp
|
||||
@@ -89,16 +86,13 @@ class IrOpPriorityConfig:
|
||||
yield IrOp.registry[field.name], op_priority
|
||||
|
||||
def set_default(self) -> None:
|
||||
"""
|
||||
Permanently set the IR op priority for all op members.
|
||||
"""
|
||||
"""Permanently set the IR op priority for all op members."""
|
||||
for ir_op, op_priority in self._iter_op_priorities():
|
||||
ir_op.set_default(op_priority)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_priority(self):
|
||||
"""
|
||||
Context manager to set the IR op priority for all op members.
|
||||
"""Context manager to set the IR op priority for all op members.
|
||||
It also imports IR kernel implementations for the current platform
|
||||
to ensure all implementations are made available.
|
||||
"""
|
||||
@@ -111,8 +105,7 @@ class IrOpPriorityConfig:
|
||||
def with_default(
|
||||
cls, default: list[str], /, **kwargs: list[str]
|
||||
) -> "IrOpPriorityConfig":
|
||||
"""
|
||||
A helper to create an IrOpPriorityConfig where fields not specified in kwargs
|
||||
"""A helper to create an IrOpPriorityConfig where fields not specified in kwargs
|
||||
use the given default list.
|
||||
"""
|
||||
for field in fields(cls): # type: ignore[arg-type]
|
||||
@@ -355,8 +348,7 @@ class KernelConfig:
|
||||
return value
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
"""
|
||||
|
||||
@@ -106,8 +106,7 @@ class KVTransferConfig:
|
||||
'fail': immediately fail the request with an error finish reason (default)"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+1
-2
@@ -118,8 +118,7 @@ class LoadConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+1
-2
@@ -86,8 +86,7 @@ class LoRAConfig:
|
||||
MoE models whose adapters use this layout; ignored otherwise."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+15
-21
@@ -418,8 +418,7 @@ class ModelConfig:
|
||||
mm_processor_device: InitVar[MMProcessorDevice | None] = None
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -938,7 +937,7 @@ class ModelConfig:
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
|
||||
"""Called after __post_init__"""
|
||||
"""Called after __post_init__."""
|
||||
if not isinstance(self.tokenizer, str):
|
||||
raise ValueError(
|
||||
f"tokenizer must be a string, got "
|
||||
@@ -1006,7 +1005,8 @@ class ModelConfig:
|
||||
|
||||
def _get_transformers_backend_cls(self) -> str:
|
||||
"""Determine which Transformers modeling backend class will be used if
|
||||
`model_impl` is set to `transformers` or `auto`."""
|
||||
`model_impl` is set to `transformers` or `auto`.
|
||||
"""
|
||||
cls = "Transformers"
|
||||
# If 'hf_config is not hf_text_config' it's a nested config, i.e. multimodal
|
||||
cls += "MultiModal" if self.hf_config is not self.hf_text_config else ""
|
||||
@@ -1083,8 +1083,8 @@ class ModelConfig:
|
||||
Args:
|
||||
model: Model name or path
|
||||
tokenizer: Tokenizer name or path
|
||||
"""
|
||||
|
||||
"""
|
||||
# Skip if model_weights is already set (model already pulled)
|
||||
if self.model_weights:
|
||||
return
|
||||
@@ -1662,9 +1662,7 @@ class ModelConfig:
|
||||
raise AssertionError(f"Unsupported block type: {block_type}")
|
||||
|
||||
def get_mamba_chunk_size(self) -> int:
|
||||
"""
|
||||
Returns the mamba chunk size if it exists
|
||||
"""
|
||||
"""Returns the mamba chunk size if it exists."""
|
||||
# used by e.g. Bamba, FalconH1, Granite
|
||||
chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None)
|
||||
if chunk_size is None:
|
||||
@@ -1679,11 +1677,11 @@ class ModelConfig:
|
||||
return chunk_size
|
||||
|
||||
def get_multimodal_config(self) -> MultiModalConfig:
|
||||
"""
|
||||
Get the multimodal configuration of the model.
|
||||
"""Get the multimodal configuration of the model.
|
||||
|
||||
Raises:
|
||||
ValueError: If the model is not multimodal.
|
||||
|
||||
"""
|
||||
if self.multimodal_config is None:
|
||||
raise ValueError("The model is not multimodal.")
|
||||
@@ -1691,8 +1689,7 @@ class ModelConfig:
|
||||
return self.multimodal_config
|
||||
|
||||
def try_get_generation_config(self) -> dict[str, Any]:
|
||||
"""
|
||||
This method attempts to retrieve the non-default values of the
|
||||
"""This method attempts to retrieve the non-default values of the
|
||||
generation config for this model.
|
||||
|
||||
The generation config can contain information about special tokens, as
|
||||
@@ -1701,6 +1698,7 @@ class ModelConfig:
|
||||
|
||||
Returns:
|
||||
A dictionary containing the non-default generation config.
|
||||
|
||||
"""
|
||||
if self.generation_config in {"auto", "vllm"}:
|
||||
config = try_get_generation_config(
|
||||
@@ -1726,8 +1724,7 @@ class ModelConfig:
|
||||
return config.to_diff_dict()
|
||||
|
||||
def get_diff_sampling_param(self) -> dict[str, Any]:
|
||||
"""
|
||||
This method returns a dictionary containing the non-default sampling
|
||||
"""This method returns a dictionary containing the non-default sampling
|
||||
parameters with `override_generation_config` applied.
|
||||
|
||||
The default sampling parameters are:
|
||||
@@ -1739,6 +1736,7 @@ class ModelConfig:
|
||||
|
||||
Returns:
|
||||
A dictionary containing the non-default sampling parameters.
|
||||
|
||||
"""
|
||||
src = self.generation_config
|
||||
|
||||
@@ -1870,8 +1868,7 @@ class ModelConfig:
|
||||
|
||||
@property
|
||||
def score_type(self) -> ScoreType:
|
||||
"""
|
||||
Scoring API handles score/rerank for:
|
||||
"""Scoring API handles score/rerank for:
|
||||
|
||||
- "classify" task (score_type: cross-encoder models)
|
||||
- "embed" task (score_type: bi-encoder models)
|
||||
@@ -1964,8 +1961,7 @@ class ModelConfig:
|
||||
|
||||
@property
|
||||
def head_dtype(self) -> torch.dtype:
|
||||
"""
|
||||
"head" refers to the last Linear layer(s) of an LLM,
|
||||
"""The "head" refers to the last Linear layer(s) of an LLM,
|
||||
such as the lm_head in a generation model,
|
||||
or the score or classifier in a classification model.
|
||||
|
||||
@@ -1976,7 +1972,6 @@ class ModelConfig:
|
||||
fp32, which is required for RL training-inference consistency
|
||||
(the trainer computes logits in fp32).
|
||||
"""
|
||||
|
||||
head_dtype = _get_head_dtype(
|
||||
config=self.hf_config, dtype=self.dtype, runner_type=self.runner_type
|
||||
)
|
||||
@@ -2174,8 +2169,7 @@ class ModelConfig:
|
||||
|
||||
|
||||
def get_served_model_name(model: str, served_model_name: str | list[str] | None):
|
||||
"""
|
||||
If the input is a non-empty list, the first model_name in
|
||||
"""If the input is a non-empty list, the first model_name in
|
||||
`served_model_name` is taken.
|
||||
If the input is a non-empty string, it is used directly.
|
||||
For cases where the input is either an empty string or an
|
||||
|
||||
@@ -14,9 +14,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
|
||||
class ModelArchitectureConfig:
|
||||
"""
|
||||
Configuration for model architecture that required by vLLM runtime
|
||||
"""
|
||||
"""Configuration for model architecture that required by vLLM runtime."""
|
||||
|
||||
architectures: list[str]
|
||||
"""List of model architecture class names (e.g., ['LlamaForCausalLM']).
|
||||
|
||||
@@ -357,6 +357,7 @@ class MultiModalConfig:
|
||||
Returns:
|
||||
The kwargs to build the config with, unchanged unless the flag adds
|
||||
a `device`.
|
||||
|
||||
"""
|
||||
if mm_processor_device in (None, "auto"):
|
||||
return mm_processor_kwargs
|
||||
@@ -390,6 +391,7 @@ class MultiModalConfig:
|
||||
ValueError: If `device` is not something `torch.device` accepts.
|
||||
`validate_mm_processor_device` is what surfaces this during
|
||||
startup, so the value is only parsed once.
|
||||
|
||||
"""
|
||||
device = (self.mm_processor_kwargs or {}).get("device")
|
||||
if device is None:
|
||||
@@ -418,6 +420,7 @@ class MultiModalConfig:
|
||||
ValueError: If the requested device is not a torch device, or if it
|
||||
is the accelerator on an instance that also runs the language
|
||||
model.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -452,8 +455,7 @@ class MultiModalConfig:
|
||||
)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -476,8 +478,7 @@ class MultiModalConfig:
|
||||
return hash_str
|
||||
|
||||
def get_limit_per_prompt(self, modality: str) -> int:
|
||||
"""
|
||||
Get the maximum number of input items allowed per prompt
|
||||
"""Get the maximum number of input items allowed per prompt
|
||||
for the given modality (backward compatible).
|
||||
"""
|
||||
if self.language_model_only:
|
||||
@@ -495,8 +496,7 @@ class MultiModalConfig:
|
||||
self,
|
||||
inference_kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Get the keyword arguments to pass to the multi-modal processor
|
||||
"""Get the keyword arguments to pass to the multi-modal processor
|
||||
according to the extra arguments passed during inference.
|
||||
"""
|
||||
kwargs = self.mm_processor_kwargs or {}
|
||||
@@ -524,7 +524,8 @@ class MultiModalConfig:
|
||||
|
||||
def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None:
|
||||
"""Return `(method, rate)` when video pruning is enabled, else None.
|
||||
`rate` is the fraction of video tokens to prune."""
|
||||
`rate` is the fraction of video tokens to prune.
|
||||
"""
|
||||
if self.video_pruning_rate is not None and self.video_pruning_rate > 0:
|
||||
return (self.video_pruning_method, float(self.video_pruning_rate))
|
||||
return None
|
||||
|
||||
@@ -112,8 +112,7 @@ class ObservabilityConfig:
|
||||
)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -157,7 +156,8 @@ class ObservabilityConfig:
|
||||
cls, value: list[DetailedTraceModules] | None
|
||||
) -> list[DetailedTraceModules] | None:
|
||||
"""Handle the legacy case where users might provide a comma-separated
|
||||
string instead of a list of strings."""
|
||||
string instead of a list of strings.
|
||||
"""
|
||||
if value is not None and len(value) == 1 and "," in value[0]:
|
||||
value = cast(list[DetailedTraceModules], value[0].split(","))
|
||||
return value
|
||||
|
||||
@@ -137,8 +137,7 @@ class OffloadConfig:
|
||||
return self
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the offload configs.
|
||||
"""Provide a hash that uniquely identifies all the offload configs.
|
||||
|
||||
All fields are included because PrefetchOffloader patches module
|
||||
forwards and inserts custom ops (wait_prefetch, start_prefetch)
|
||||
|
||||
@@ -602,15 +602,13 @@ class ParallelConfig:
|
||||
|
||||
@property
|
||||
def local_engines_only(self) -> bool:
|
||||
"""
|
||||
Client manages local+remote EngineCores in pure internal LB case.
|
||||
"""Client manages local+remote EngineCores in pure internal LB case.
|
||||
Client manages local EngineCores in hybrid and external LB case.
|
||||
"""
|
||||
return self.data_parallel_external_lb or self.data_parallel_hybrid_lb
|
||||
|
||||
def get_next_dp_init_port(self) -> int:
|
||||
"""
|
||||
We might need to initialize process groups in multiple
|
||||
"""We might need to initialize process groups in multiple
|
||||
processes that is related to data parallelism,
|
||||
e.g. both in the worker and in the engine, which
|
||||
can live in different processes. To avoid port conflicts, we
|
||||
@@ -794,6 +792,7 @@ class ParallelConfig:
|
||||
|
||||
Returns:
|
||||
(has_unfinished_global, pause_consensus)
|
||||
|
||||
"""
|
||||
tensor = torch.tensor(
|
||||
[int(has_unfinished), int(pending_pause)], dtype=torch.int32, device="cpu"
|
||||
@@ -815,8 +814,7 @@ class ParallelConfig:
|
||||
return tensor.item()
|
||||
|
||||
def compute_hash(self):
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
|
||||
@@ -176,8 +176,7 @@ class PoolerConfig:
|
||||
return self.tok_pooling_type
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -151,8 +151,7 @@ class ProfilerConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -47,13 +47,15 @@ class ReasoningConfig:
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""Returns True if reasoning is enabled (i.e. if token IDs have been
|
||||
initialized), False otherwise."""
|
||||
initialized), False otherwise.
|
||||
"""
|
||||
return self._enabled
|
||||
|
||||
@property
|
||||
def reasoning_start_token_ids(self) -> list[int] | None:
|
||||
"""Token IDs derived from `reasoning_start_str`. Set automatically by
|
||||
`initialize_token_ids`. Not intended to be configured directly."""
|
||||
`initialize_token_ids`. Not intended to be configured directly.
|
||||
"""
|
||||
return self._reasoning_start_token_ids
|
||||
|
||||
@property
|
||||
|
||||
@@ -200,9 +200,7 @@ class SchedulerConfig:
|
||||
|
||||
@staticmethod
|
||||
def default_factory(**kwargs):
|
||||
"""
|
||||
Factory method to create `SchedulerConfig` with default values for `InitVar`s.
|
||||
"""
|
||||
"""Create a `SchedulerConfig` with default values for its `InitVar`s."""
|
||||
if "max_model_len" not in kwargs:
|
||||
kwargs["max_model_len"] = 8192
|
||||
if "is_encoder_decoder" not in kwargs:
|
||||
@@ -233,8 +231,7 @@ class SchedulerConfig:
|
||||
return resolve_obj_by_qualname(self.scheduler_cls)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -543,7 +543,8 @@ class SpeculativeConfig:
|
||||
@staticmethod
|
||||
def _acceptance_length_to_rates(length: float, n: int) -> list[float]:
|
||||
"""Mean acceptance length to unconditional per-position rates, using
|
||||
the minimum-variance schedule."""
|
||||
the minimum-variance schedule.
|
||||
"""
|
||||
num_drafts = length - 1 # expected number of accepted draft tokens
|
||||
num_full = int(num_drafts)
|
||||
return (
|
||||
@@ -557,7 +558,8 @@ class SpeculativeConfig:
|
||||
length: float | None,
|
||||
) -> list[float]:
|
||||
"""Return per-position unconditional acceptance rates from exactly one
|
||||
of `rates` or `length` (validates range, length, and monotonicity)."""
|
||||
of `rates` or `length` (validates range, length, and monotonicity).
|
||||
"""
|
||||
if (rates is None) == (length is None):
|
||||
raise ValueError(
|
||||
"rejection_sample_method='synthetic' requires exactly one of "
|
||||
@@ -598,8 +600,7 @@ class SpeculativeConfig:
|
||||
top-k base-logit candidates. Requires draft tensor parallel size 1."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -1627,7 +1628,6 @@ class SpeculativeConfig:
|
||||
speculative_max_model_len is mainly used for testing that sequences can
|
||||
skip speculation.
|
||||
"""
|
||||
|
||||
if speculative_max_model_len is not None:
|
||||
if speculative_max_model_len > draft_max_model_len:
|
||||
raise ValueError(
|
||||
@@ -1671,6 +1671,7 @@ class SpeculativeConfig:
|
||||
Args:
|
||||
draft_hf_config: The draft model's HF config, mutated in place.
|
||||
target_max_model_len: The target model's max_model_len.
|
||||
|
||||
"""
|
||||
draft_max_position_embeddings = getattr(
|
||||
draft_hf_config, "max_position_embeddings", None
|
||||
@@ -1695,8 +1696,7 @@ class SpeculativeConfig:
|
||||
speculative_draft_tensor_parallel_size: int | None,
|
||||
draft_hf_config: PretrainedConfig,
|
||||
) -> int:
|
||||
"""
|
||||
Verifies and adjusts the tensor parallel size for a draft model
|
||||
"""Verifies and adjusts the tensor parallel size for a draft model
|
||||
specified using speculative_draft_tensor_parallel_size.
|
||||
"""
|
||||
# If speculative_draft_tensor_parallel_size is unset then set it
|
||||
@@ -1725,8 +1725,7 @@ class SpeculativeConfig:
|
||||
return speculative_draft_tensor_parallel_size
|
||||
|
||||
def update_arch_(self):
|
||||
"""
|
||||
EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
|
||||
"""EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
|
||||
architectures-related fields in self.draft_model_config
|
||||
"""
|
||||
self.draft_model_config.hf_text_config = get_hf_text_config(
|
||||
|
||||
@@ -42,8 +42,7 @@ class StructuredOutputsConfig:
|
||||
"""Whether to use structured input for reasoning."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+14
-14
@@ -64,7 +64,9 @@ def config(
|
||||
cls: The class to decorate
|
||||
config: The pydantic ConfigDict to use. If provided, it will be merged with
|
||||
the default config.
|
||||
**kwargs: Additional arguments to pass to pydantic.dataclass."""
|
||||
**kwargs: Additional arguments to pass to pydantic.dataclass.
|
||||
|
||||
"""
|
||||
# Extra fields are forbidden by default
|
||||
merged_config = ConfigDict(extra="forbid")
|
||||
if config is not None:
|
||||
@@ -82,7 +84,8 @@ def config(
|
||||
|
||||
def get_field(cls: ConfigType, name: str) -> Any:
|
||||
"""Get the default factory field of a dataclass by name. Used for getting
|
||||
default factory fields in `EngineArgs`."""
|
||||
default factory fields in `EngineArgs`.
|
||||
"""
|
||||
if not is_dataclass(cls):
|
||||
raise TypeError("The given class is not a dataclass.")
|
||||
try:
|
||||
@@ -119,7 +122,8 @@ def is_init_field(cls: ConfigType, name: str) -> bool:
|
||||
def replace(dataclass_instance: ConfigT, /, **kwargs) -> ConfigT:
|
||||
"""Like [`dataclasses.replace`](https://docs.python.org/3/library/dataclasses.html#dataclasses.replace),
|
||||
but compatible with Pydantic dataclasses which use `pydantic.fields.Field` instead
|
||||
of `dataclasses.field`"""
|
||||
of `dataclasses.field`
|
||||
"""
|
||||
cls = type(dataclass_instance)
|
||||
dataclass_dict = dataclass_instance.__dict__
|
||||
dataclass_dict = {k: v for k, v in dataclass_dict.items() if is_init_field(cls, k)}
|
||||
@@ -134,8 +138,7 @@ def getattr_iter(
|
||||
default_factory: Callable[[], Any] | None = None,
|
||||
warn: bool = False,
|
||||
) -> Any:
|
||||
"""
|
||||
A helper function that retrieves an attribute from an object which may
|
||||
"""A helper function that retrieves an attribute from an object which may
|
||||
have multiple possible names. This is useful when fetching attributes from
|
||||
arbitrary `transformers.PretrainedConfig` instances.
|
||||
|
||||
@@ -158,12 +161,10 @@ def getattr_iter(
|
||||
|
||||
|
||||
def get_attr_docs(cls: type[Any]) -> dict[str, str]:
|
||||
"""
|
||||
Get any docstrings placed after attribute assignments in a class body.
|
||||
"""Get any docstrings placed after attribute assignments in a class body.
|
||||
|
||||
https://davidism.com/mit-license/
|
||||
"""
|
||||
|
||||
cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
|
||||
|
||||
if not isinstance(cls_node, ast.ClassDef):
|
||||
@@ -385,8 +386,7 @@ def hash_factors(items: dict[str, object]) -> str:
|
||||
|
||||
@dataclass
|
||||
class Range:
|
||||
"""
|
||||
A range of numbers.
|
||||
"""A range of numbers.
|
||||
Inclusive of start, inclusive of end.
|
||||
"""
|
||||
|
||||
@@ -445,8 +445,7 @@ def get_from_deprecated_env_if_set(
|
||||
removal_version: str,
|
||||
field_name: str | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
Get value from deprecated environment variable with warning.
|
||||
"""Get value from deprecated environment variable with warning.
|
||||
|
||||
Args:
|
||||
env_name: Name of the deprecated environment variable
|
||||
@@ -455,6 +454,7 @@ def get_from_deprecated_env_if_set(
|
||||
|
||||
Returns:
|
||||
The environment variable value if set, None otherwise
|
||||
|
||||
"""
|
||||
if envs.is_set(env_name):
|
||||
value = os.environ.get(env_name)
|
||||
@@ -477,8 +477,7 @@ def set_from_deprecated_env_if_set(
|
||||
to_bool: bool = False,
|
||||
to_int: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Set object field from deprecated environment variable with warning.
|
||||
"""Set object field from deprecated environment variable with warning.
|
||||
|
||||
Args:
|
||||
config: Config object to set the field on
|
||||
@@ -489,6 +488,7 @@ def set_from_deprecated_env_if_set(
|
||||
to_int: Whether to convert the environment variable value to integer
|
||||
Returns:
|
||||
None
|
||||
|
||||
"""
|
||||
if to_bool and to_int:
|
||||
raise ValueError("Cannot convert to both boolean and integer.")
|
||||
|
||||
+19
-28
@@ -153,8 +153,8 @@ IS_DENSE = False
|
||||
|
||||
def enable_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if either RMS norm or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion."""
|
||||
|
||||
otherwise Inductor handles fusion.
|
||||
"""
|
||||
return (
|
||||
cfg.compilation_config.is_custom_op_enabled("rms_norm")
|
||||
or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
|
||||
@@ -163,8 +163,7 @@ def enable_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
|
||||
def enable_act_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""
|
||||
Enable if either SiLU+Mul or quant FP8 custom op is active;
|
||||
"""Enable if either SiLU+Mul or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion.
|
||||
Also enable for FP4 models as FP4 quant is always custom so Inductor cannot fuse it.
|
||||
"""
|
||||
@@ -220,7 +219,6 @@ def enable_rope_kvcache_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if use_inductor_graph_partition is enabled."""
|
||||
|
||||
return (
|
||||
cfg.compilation_config.use_inductor_graph_partition
|
||||
or not cfg.compilation_config.splitting_ops_contain_kv_cache_update()
|
||||
@@ -229,7 +227,6 @@ def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
def enable_norm_pad_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if using AITER RMSNorm and hidden size is 2880 i.e. gpt-oss."""
|
||||
|
||||
return (
|
||||
cfg.kernel_config.ir_op_priority.fused_add_rms_norm[0] == "aiter"
|
||||
and cfg.model_config is not None
|
||||
@@ -460,8 +457,7 @@ class VllmConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self, include_version: bool = True) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -473,6 +469,7 @@ class VllmConfig:
|
||||
|
||||
Args:
|
||||
include_version: Include the vLLM version in the hash.
|
||||
|
||||
"""
|
||||
factors: list[Any] = []
|
||||
|
||||
@@ -747,7 +744,8 @@ class VllmConfig:
|
||||
|
||||
def _is_dflash2_draft(self) -> bool:
|
||||
"""Whether the DFlash draft is a DFlash2 one, by the architecture the
|
||||
speculator selects on (v1/worker/gpu/spec_decode/__init__.py)."""
|
||||
speculator selects on (v1/worker/gpu/spec_decode/__init__.py).
|
||||
"""
|
||||
spec = self.speculative_config
|
||||
if spec is None or spec.method != "dflash":
|
||||
return False
|
||||
@@ -798,8 +796,7 @@ class VllmConfig:
|
||||
|
||||
@property
|
||||
def needs_dp_coordinator(self) -> bool:
|
||||
"""
|
||||
Determine if the DPCoordinator process is needed.
|
||||
"""Determine if the DPCoordinator process is needed.
|
||||
|
||||
The DPCoordinator is needed in two cases:
|
||||
1. For MoE models with DP > 1: to handle wave coordination
|
||||
@@ -809,8 +806,8 @@ class VllmConfig:
|
||||
|
||||
Returns:
|
||||
True if DPCoordinator process is needed, False otherwise.
|
||||
"""
|
||||
|
||||
"""
|
||||
# For non-MoE models, only need coordinator in internal/hybrid LB mode
|
||||
# (for stats collection).
|
||||
return self.parallel_config.data_parallel_size > 1 and (
|
||||
@@ -820,8 +817,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
def enable_trace_function_call_for_thread(self) -> None:
|
||||
"""
|
||||
Set up function tracing for the current thread,
|
||||
"""Set up function tracing for the current thread,
|
||||
if enabled via the `VLLM_TRACE_FUNCTION` environment variable.
|
||||
"""
|
||||
if envs.VLLM_TRACE_FUNCTION:
|
||||
@@ -955,6 +951,7 @@ class VllmConfig:
|
||||
config_obj: Configuration object to update.
|
||||
key: Attribute name.
|
||||
value: Default value (static or callable).
|
||||
|
||||
"""
|
||||
if getattr(config_obj, key) is None:
|
||||
# Some config values are known before initialization and are
|
||||
@@ -976,6 +973,7 @@ class VllmConfig:
|
||||
|
||||
Args:
|
||||
defaults: Dictionary of default values to apply.
|
||||
|
||||
"""
|
||||
|
||||
def apply_recursive(config_obj: Any, config_defaults: dict[str, Any]) -> None:
|
||||
@@ -1276,7 +1274,6 @@ class VllmConfig:
|
||||
|
||||
def __post_init__(self):
|
||||
"""Verify configs are valid & consistent with each other."""
|
||||
|
||||
# To give each torch profile run a unique instance name.
|
||||
self.instance_id = f"{time.time_ns()}"
|
||||
|
||||
@@ -2133,8 +2130,7 @@ class VllmConfig:
|
||||
]
|
||||
|
||||
def _set_max_num_scheduled_tokens(self):
|
||||
"""
|
||||
In most cases, the scheduler may schedule a batch with as many tokens as the
|
||||
"""In most cases, the scheduler may schedule a batch with as many tokens as the
|
||||
worker is configured to handle.
|
||||
"""
|
||||
if self.speculative_config is not None:
|
||||
@@ -2172,8 +2168,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
def _set_cudagraph_sizes(self):
|
||||
"""
|
||||
vLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
"""VLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
capture as:
|
||||
|
||||
```python
|
||||
@@ -2219,8 +2214,8 @@ class VllmConfig:
|
||||
padded CUDA graph will be used.
|
||||
- If batch size > largest `cudagraph_capture_sizes`, cudagraph will
|
||||
not be used.
|
||||
"""
|
||||
|
||||
"""
|
||||
if (
|
||||
self.model_config is not None
|
||||
and not self.model_config.enforce_eager
|
||||
@@ -2441,9 +2436,7 @@ class VllmConfig:
|
||||
self.compilation_config.post_init_cudagraph_sizes()
|
||||
|
||||
def _set_compile_ranges(self):
|
||||
"""
|
||||
Set the compile ranges for the compilation config.
|
||||
"""
|
||||
"""Set the compile ranges for the compilation config."""
|
||||
compilation_config = self.compilation_config
|
||||
computed_compile_ranges_endpoints = []
|
||||
|
||||
@@ -3310,8 +3303,7 @@ _current_prefix: str | None = None
|
||||
def set_current_vllm_config(
|
||||
vllm_config: VllmConfig, check_compile=False, prefix: str | None = None
|
||||
):
|
||||
"""
|
||||
Temporarily set the current vLLM config.
|
||||
"""Temporarily set the current vLLM config.
|
||||
Used during model initialization.
|
||||
We save the current vLLM config in a global variable,
|
||||
so that all modules can access it, e.g. custom ops
|
||||
@@ -3392,15 +3384,14 @@ def get_layers_from_vllm_config(
|
||||
layer_type: type[T],
|
||||
layer_names: Iterable[str] | None = None,
|
||||
) -> dict[str, T]:
|
||||
"""
|
||||
Get layers from the vLLM config.
|
||||
"""Get layers from the vLLM config.
|
||||
|
||||
Args:
|
||||
vllm_config: The vLLM config.
|
||||
layer_type: The type of the layer to get.
|
||||
layer_names: The names of the layers to get. If None, return all layers.
|
||||
"""
|
||||
|
||||
"""
|
||||
forward_context = vllm_config.compilation_config.static_forward_context
|
||||
if layer_names is None:
|
||||
layer_names = forward_context.keys()
|
||||
|
||||
@@ -80,8 +80,7 @@ def use_memory_pool_with_allocator(
|
||||
|
||||
|
||||
class CuMemAllocator:
|
||||
"""
|
||||
A singleton class that manages a memory pool for CUDA tensors.
|
||||
"""A singleton class that manages a memory pool for CUDA tensors.
|
||||
The memory in this pool can be offloaded or discarded when the
|
||||
allocator sleeps.
|
||||
|
||||
@@ -109,8 +108,7 @@ class CuMemAllocator:
|
||||
|
||||
@staticmethod
|
||||
def get_instance() -> "CuMemAllocator":
|
||||
"""
|
||||
CuMemAllocator is a singleton class.
|
||||
"""CuMemAllocator is a singleton class.
|
||||
We cannot call the constructor directly.
|
||||
Call this method to get the instance.
|
||||
"""
|
||||
@@ -182,9 +180,9 @@ class CuMemAllocator:
|
||||
self.release_pools()
|
||||
|
||||
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
|
||||
"""Internal method to store the allocation data
|
||||
when memory is allocated in the memory pool.
|
||||
"""
|
||||
Internal method to store the allocation data
|
||||
when memory is allocated in the memory pool."""
|
||||
py_d_mem = allocation_handle[2]
|
||||
self.pointer_to_data[py_d_mem] = AllocationData(
|
||||
allocation_handle, self.current_tag
|
||||
@@ -198,9 +196,9 @@ class CuMemAllocator:
|
||||
return
|
||||
|
||||
def _python_free_callback(self, ptr: int) -> HandleType:
|
||||
"""Internal method to look up the allocation data
|
||||
when memory is freed in the memory pool.
|
||||
"""
|
||||
Internal method to look up the allocation data
|
||||
when memory is freed in the memory pool."""
|
||||
data = self.pointer_to_data.pop(ptr)
|
||||
if data.cpu_backup_tensor is not None:
|
||||
data.cpu_backup_tensor = None
|
||||
@@ -227,14 +225,14 @@ class CuMemAllocator:
|
||||
return data.handle
|
||||
|
||||
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
|
||||
"""
|
||||
Put the allocator in sleep mode.
|
||||
"""Put the allocator in sleep mode.
|
||||
All data in the memory allocation with the specified tag will be
|
||||
offloaded to CPU memory, and others will be discarded.
|
||||
|
||||
Args:
|
||||
offload_tags: The tags of the memory allocation that will be
|
||||
offloaded. The rest of the memory allocation will be discarded.
|
||||
|
||||
"""
|
||||
if offload_tags is None:
|
||||
# by default, allocated tensors are offloaded
|
||||
@@ -325,8 +323,7 @@ class CuMemAllocator:
|
||||
)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
"""
|
||||
Wake up the allocator from sleep mode.
|
||||
"""Wake up the allocator from sleep mode.
|
||||
All data that is previously offloaded will be loaded back to GPU
|
||||
memory, and the rest of the data will have empty memory.
|
||||
|
||||
@@ -334,6 +331,7 @@ class CuMemAllocator:
|
||||
tags: The tags of the memory allocation that will be loaded
|
||||
back to GPU memory. If None, all memory allocation will be loaded
|
||||
back to GPU memory.
|
||||
|
||||
"""
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -357,14 +355,14 @@ class CuMemAllocator:
|
||||
|
||||
@contextmanager
|
||||
def use_memory_pool(self, tag: str | None = None):
|
||||
"""
|
||||
A context manager to use the memory pool.
|
||||
"""A context manager to use the memory pool.
|
||||
All memory allocation created inside the context will be allocated
|
||||
in the memory pool, and has the specified tag.
|
||||
|
||||
Args:
|
||||
tag: The tag of the memory allocation. If None, the default tag
|
||||
will be used.
|
||||
|
||||
"""
|
||||
if tag is None:
|
||||
tag = CuMemAllocator.default_tag
|
||||
@@ -416,9 +414,7 @@ class CuMemAllocator:
|
||||
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
|
||||
|
||||
def get_current_usage(self) -> int:
|
||||
"""
|
||||
Get the total number of bytes allocated in the memory pool.
|
||||
"""
|
||||
"""Get the total number of bytes allocated in the memory pool."""
|
||||
sum_bytes: int = 0
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
handle = data.handle
|
||||
|
||||
@@ -70,7 +70,8 @@ class SleepModeBackend(ABC):
|
||||
|
||||
def state(self) -> SleepModeState:
|
||||
"""Current lifecycle state. Lets ``/health`` distinguish a healthy-idle
|
||||
(suspended) engine from a healthy-serving one (see RFC #34303)."""
|
||||
(suspended) engine from a healthy-serving one (see RFC #34303).
|
||||
"""
|
||||
return self._state
|
||||
|
||||
# -- Capability introspection (no instance required) --
|
||||
@@ -83,26 +84,30 @@ class SleepModeBackend(ABC):
|
||||
@classmethod
|
||||
def preserves_communicators(cls) -> bool:
|
||||
"""If False, collective communicators (e.g. NCCL) are destroyed by
|
||||
``suspend`` and the executor must re-initialize them on ``resume``."""
|
||||
``suspend`` and the executor must re-initialize them on ``resume``.
|
||||
"""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def preserves_compiled_artifacts(cls) -> bool:
|
||||
"""If True, torch.compile / JIT kernels survive suspend/resume and need
|
||||
not be recompiled on resume."""
|
||||
not be recompiled on resume.
|
||||
"""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def preserves_graphs_with_communicators(cls) -> bool:
|
||||
"""If True, CUDA graphs containing collective communicators (e.g. NCCL)
|
||||
stay valid after resume. False when communicators are rebuilt (embedded
|
||||
comm handles go stale)."""
|
||||
comm handles go stale).
|
||||
"""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_durable_storage(cls) -> bool:
|
||||
"""If True, suspended state can be persisted beyond the process
|
||||
lifetime (disk or object storage) and restored in a new process."""
|
||||
lifetime (disk or object storage) and restored in a new process.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -116,6 +116,7 @@ class XpuMemAllocator:
|
||||
Sleep will offload selected payloads to CPU or discard and unmap XPU
|
||||
physical memory. Wake-up remaps physical memory back to the same
|
||||
reserved virtual address and restores payload.
|
||||
|
||||
"""
|
||||
|
||||
instance: "XpuMemAllocator | None" = None
|
||||
|
||||
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
|
||||
|
||||
@classmethod
|
||||
def effective_max_size(cls) -> int:
|
||||
"""
|
||||
Max input byte size eligible for AITER custom allreduce.
|
||||
"""
|
||||
"""Max input byte size eligible for AITER custom allreduce."""
|
||||
return cls.MAX_SIZE // 2
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
An implementation of all2all communication based on
|
||||
"""An implementation of all2all communication based on
|
||||
all-gather (dispatch) and reduce-scatter (combine).
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Reduce-scatter hidden_states across all dp ranks.
|
||||
"""
|
||||
"""Reduce-scatter hidden_states across all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(
|
||||
hidden_states.shape[0] // dist_group.world_size,
|
||||
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
assert has_deep_ep(), (
|
||||
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP Low-Latency kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP Low-Latency kernels."""
|
||||
|
||||
_buffer: Any = None
|
||||
_mask: torch.Tensor | None = None
|
||||
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
num_global_experts: int,
|
||||
num_local_experts: int,
|
||||
) -> dict[Any, Any]:
|
||||
"""
|
||||
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
can dispatch all the ranks must hold the same value.
|
||||
token_hidden_size: the hidden dimension of each token.
|
||||
num_ep_ranks: the number of EP group ranks.
|
||||
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
return kwargs
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
"""
|
||||
The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
"""The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
_make_all2all_kwargs.
|
||||
"""
|
||||
import deep_ep # type: ignore[import-not-found]
|
||||
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
|
||||
|
||||
|
||||
class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on NIXL EP kernels.
|
||||
"""All2All communication based on NIXL EP kernels.
|
||||
This backend supports elastic EP with dynamic rank connection/disconnection.
|
||||
"""
|
||||
|
||||
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
|
||||
"""
|
||||
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
|
||||
|
||||
# This type lint could be removed after all of the work in
|
||||
# https://github.com/vllm-project/vllm/issues/26533 done.
|
||||
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
rank: int,
|
||||
gpus_per_node: int,
|
||||
):
|
||||
"""Initialize workspace"""
|
||||
"""Initialize workspace."""
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
)
|
||||
|
||||
def ensure_alltoall_workspace_initialized(self):
|
||||
"""Ensure workspace is initialized"""
|
||||
"""Ensure workspace is initialized."""
|
||||
if not has_flashinfer_nvlink_two_sided():
|
||||
return False
|
||||
|
||||
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
return self
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up workspace"""
|
||||
"""Clean up workspace."""
|
||||
if (
|
||||
self.initialized
|
||||
and self.workspace_tensor is not None
|
||||
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
This is a newer kernel from trtllm that should perform better than the kernel
|
||||
used by flashinfer_nvlink_two_sided.
|
||||
"""
|
||||
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
Uses NCCL Gin backend with analytical SM calculation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
|
||||
|
||||
|
||||
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
|
||||
"""
|
||||
Determine if NCCL symmetric memory allreduce should be used.
|
||||
"""Determine if NCCL symmetric memory allreduce should be used.
|
||||
|
||||
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
|
||||
- Small tensors (≤16K): Lower latency than custom_AR
|
||||
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
|
||||
|
||||
def should_nccl_symm_mem_ag_rs() -> bool:
|
||||
"""Check whether NCCL symmetric memory should be used for
|
||||
AllGather / ReduceScatter collectives."""
|
||||
AllGather / ReduceScatter collectives.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
is_symmetric_memory_enabled,
|
||||
)
|
||||
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
|
||||
|
||||
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
"""Check if GPU src can access GPU tgt."""
|
||||
|
||||
# if the cache variable is already calculated,
|
||||
# read from the cache instead of checking it again
|
||||
global _gpu_p2p_access_cache
|
||||
|
||||
@@ -114,6 +114,7 @@ class All2AllManagerBase:
|
||||
Returns:
|
||||
An int32 device tensor where 0 marks a live rank and 1 marks a
|
||||
masked (dead/unreachable) rank.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -171,8 +172,7 @@ class All2AllManagerBase:
|
||||
|
||||
|
||||
class DeviceCommunicatorBase:
|
||||
"""
|
||||
Base class for device-specific communicator.
|
||||
"""Base class for device-specific communicator.
|
||||
It can use the `cpu_group` to initialize the communicator.
|
||||
If the device has PyTorch integration (PyTorch can recognize its
|
||||
communication backend), the `device_group` will also be given.
|
||||
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
|
||||
return output_tensor
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
return hidden_states
|
||||
|
||||
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
logger.info("Using allgather_reducescatter all2all manager.")
|
||||
|
||||
def _all_group_ranks_share_shm_group_name(self) -> bool:
|
||||
"""
|
||||
CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
|
||||
"""
|
||||
local_name = _CPUSHMDistributed.make_group_name(self)
|
||||
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return symm_output
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -129,8 +129,7 @@ class CustomAllreduce:
|
||||
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
|
||||
symm_mem_enabled=False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the CustomAllreduce to. If None,
|
||||
@@ -138,6 +137,7 @@ class CustomAllreduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self._IS_CAPTURING = False
|
||||
self._ptr = 0
|
||||
@@ -370,8 +370,7 @@ class CustomAllreduce:
|
||||
|
||||
@contextmanager
|
||||
def capture(self):
|
||||
"""
|
||||
The main responsibility of this context manager is the
|
||||
"""The main responsibility of this context manager is the
|
||||
`register_graph_buffers` call at the end of the context.
|
||||
It records all the buffer addresses used in the CUDA graph.
|
||||
"""
|
||||
|
||||
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
||||
is True only when ``auto`` selects mnnvl for a single node, so that
|
||||
workspace creation can fall back to trtllm on single-node topologies
|
||||
without NVSwitch multicast support (where mnnvl is unavailable).
|
||||
|
||||
"""
|
||||
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
||||
if backend != "auto":
|
||||
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
|
||||
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
|
||||
for standalone allreduce. Backend is controlled by
|
||||
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
|
||||
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
|
||||
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
|
||||
|
||||
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the PyNcclCommunicator to. If None,
|
||||
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
|
||||
use the default library path.
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device.
|
||||
|
||||
"""
|
||||
if not isinstance(group, StatelessProcessGroup):
|
||||
assert dist.is_initialized()
|
||||
|
||||
@@ -62,8 +62,7 @@ class QuickAllReduce:
|
||||
}
|
||||
|
||||
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
|
||||
"""
|
||||
Custom allreduce provides non-destructive acceleration and is
|
||||
"""Custom allreduce provides non-destructive acceleration and is
|
||||
available for CUDA and ROCm MI300 series.
|
||||
|
||||
Custom quick allreduce leverages quantization for further
|
||||
@@ -86,6 +85,7 @@ class QuickAllReduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self.disabled = True
|
||||
if not self._rocm_arch_available():
|
||||
@@ -301,8 +301,7 @@ class QuickAllReduce:
|
||||
return False
|
||||
|
||||
def create_shared_buffer(self):
|
||||
"""
|
||||
Creates a shared buffer for quickreduce.
|
||||
"""Creates a shared buffer for quickreduce.
|
||||
Has to be called after init_custom_qr
|
||||
"""
|
||||
handle = ops.qr_get_handle(self._ptr)
|
||||
@@ -312,9 +311,7 @@ class QuickAllReduce:
|
||||
ops.qr_open_handles(self._ptr, handles)
|
||||
|
||||
def should_quick_allreduce(self, inp: torch.Tensor):
|
||||
"""
|
||||
Check if quickreduce is available
|
||||
"""
|
||||
"""Check if quickreduce is available."""
|
||||
if self.disabled:
|
||||
return False
|
||||
if inp.dtype not in self._SUPPORTED_DTYPES:
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RayPPCommunicator(Communicator):
|
||||
"""
|
||||
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
This is wraps around the vLLM _PP GroupCoordinator.
|
||||
|
||||
This class is not thread-safe.
|
||||
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
|
||||
cuda_stream: torch.cuda.Stream | None,
|
||||
use_communication_streams: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize a RayPPCommunicator that can be used to communicate with
|
||||
"""Initialize a RayPPCommunicator that can be used to communicate with
|
||||
other Ray Compiled Graph actors for pipeline parallelism.
|
||||
|
||||
Args:
|
||||
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
|
||||
is not supported.
|
||||
use_communication_streams: Whether to use communication streams.
|
||||
This is not supported.
|
||||
|
||||
"""
|
||||
self._world_size = world_size
|
||||
self._rank: int | None = None
|
||||
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
|
||||
self._closed = False
|
||||
|
||||
def _build_actor_rank_mapping(self):
|
||||
"""
|
||||
Use collective communication to build a mapping from actor IDs to ranks.
|
||||
"""Use collective communication to build a mapping from actor IDs to ranks.
|
||||
This should be called once during initialization.
|
||||
"""
|
||||
if self._comm is None:
|
||||
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
|
||||
return self._actor_handles
|
||||
|
||||
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
|
||||
"""
|
||||
Return the given actor's rank using device communicator collective ops.
|
||||
"""
|
||||
"""Return the given actor's rank using device communicator collective ops."""
|
||||
assert hasattr(self, "_actor_id_to_rank"), (
|
||||
"Actor rank mapping not built. "
|
||||
"This should have been done during initialization."
|
||||
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
|
||||
raise ValueError(f"Actor {actor} not found in communicator group")
|
||||
|
||||
def get_self_rank(self) -> int | None:
|
||||
"""
|
||||
Return this actor's rank.
|
||||
"""
|
||||
"""Return this actor's rank."""
|
||||
return self._rank
|
||||
|
||||
def get_world_size(self) -> int:
|
||||
"""
|
||||
Return the number of ranks in the RayPPCommunicator group.
|
||||
"""
|
||||
"""Return the number of ranks in the RayPPCommunicator group."""
|
||||
return self._world_size
|
||||
|
||||
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
|
||||
"""
|
||||
Send a torch.Tensor to a peer.
|
||||
"""Send a torch.Tensor to a peer.
|
||||
|
||||
This returns when the send kernel has been queued, but the kernel may
|
||||
not have completed. Therefore, the caller should ensure that there are
|
||||
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
|
||||
buf: The torch.Tensor to send. It should already be on this
|
||||
actor's default device.
|
||||
peer_rank: The rank of the actor to send to.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: int,
|
||||
allocator: TorchTensorAllocator,
|
||||
) -> "torch.Tensor":
|
||||
"""
|
||||
Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
"""Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
|
||||
After this call returns, the receive buffer is safe to read from
|
||||
any stream. An RayChannelError will be raised if an error occurred
|
||||
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: The rank of the actor to receive from.
|
||||
allocator: The allocator to use to create the received tensor.
|
||||
This is ignored for this implementation.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
|
||||
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
|
||||
|
||||
|
||||
def memory_fence():
|
||||
"""
|
||||
Full memory barrier for shared memory synchronization.
|
||||
"""Full memory barrier for shared memory synchronization.
|
||||
|
||||
Ensures all prior memory writes are visible to other processes before
|
||||
any subsequent reads. This is critical for lock-free producer-consumer
|
||||
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
|
||||
|
||||
|
||||
class SpinCondition:
|
||||
"""
|
||||
This class implements an interface similar to a threading.Condition. It
|
||||
"""This class implements an interface similar to a threading.Condition. It
|
||||
allows a writer to notify readers to wake up and read from the shared memory
|
||||
buffer. This notification is done over a zmq socket.
|
||||
|
||||
@@ -241,6 +239,7 @@ def check_shm_free_space(
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the SHM filesystem has insufficient space.
|
||||
|
||||
"""
|
||||
if os.path.isdir(shm_path):
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
@@ -651,7 +650,8 @@ class MessageQueue:
|
||||
|
||||
def shutdown(self):
|
||||
"""If this is an idle reader, wakes it up so it can clean up and shut
|
||||
down"""
|
||||
down
|
||||
"""
|
||||
self.shutting_down = True
|
||||
if self._spin_condition is not None:
|
||||
self._spin_condition.cancel()
|
||||
@@ -761,7 +761,8 @@ class MessageQueue:
|
||||
|
||||
def should_warn(self) -> bool:
|
||||
"""Returns true if it's time to log a warning for a timeout that is not
|
||||
indefinite"""
|
||||
indefinite
|
||||
"""
|
||||
if self._should_warn:
|
||||
elapsed = time.monotonic() - self.started
|
||||
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
|
||||
@@ -941,8 +942,7 @@ class MessageQueue:
|
||||
reader_rank: int = 0,
|
||||
blocking: bool = False,
|
||||
) -> tuple["MessageQueue", list[Handle]]:
|
||||
"""
|
||||
Creates a MessageQueue for a process group with a single reader.
|
||||
"""Creates a MessageQueue for a process group with a single reader.
|
||||
|
||||
This method is designed for scenarios where only one process (the reader)
|
||||
will consume messages, and all other processes are writers. It sets up
|
||||
@@ -962,6 +962,7 @@ class MessageQueue:
|
||||
tuple[MessageQueue, list[Handle]]:
|
||||
The MessageQueue instance for the calling process,
|
||||
and a list of handles (only non-empty for the reader process).
|
||||
|
||||
"""
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
@@ -994,8 +995,7 @@ class MessageQueue:
|
||||
external_writer_handle=None,
|
||||
blocking: bool = True,
|
||||
) -> "MessageQueue":
|
||||
"""
|
||||
Creates a MessageQueue for a distributed process group with one writer and
|
||||
"""Creates a MessageQueue for a distributed process group with one writer and
|
||||
multiple readers.
|
||||
|
||||
This method is designed for scenarios where one process (the writer) sends
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleWriterShmRingBuffer:
|
||||
"""
|
||||
A single-writer, multiple-reader ring buffer implementation using shared
|
||||
"""A single-writer, multiple-reader ring buffer implementation using shared
|
||||
memory. This class provides a thread-safe ring buffer where one process
|
||||
can write data while multiple processes/threads can read from it.
|
||||
|
||||
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
|
||||
return int.from_bytes(byte_data, "little", signed=True)
|
||||
|
||||
def allocate_buf(self, size: int) -> tuple[int, int]:
|
||||
"""
|
||||
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
Memory layout:
|
||||
`[4-byte monotonic_id][4-byte size][buffer data...]`
|
||||
"""
|
||||
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
|
||||
is_free_fn: Callable[[int, memoryview], bool],
|
||||
nbytes: int | None = None,
|
||||
) -> Iterable[int]:
|
||||
"""
|
||||
Free a buffer of the given size. This is a no-op in shared memory,
|
||||
"""Free a buffer of the given size. This is a no-op in shared memory,
|
||||
but we need to keep track of the metadata.
|
||||
|
||||
If freed memory spreads across the end and start of the ring buffer,
|
||||
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
|
||||
still might not be a contiguous space of `nbytes` available.
|
||||
|
||||
Args:
|
||||
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
|
||||
a monotonic id and the buffer, returning True when that buffer
|
||||
can be reclaimed.
|
||||
nbytes (int, optional): The size of the buffer to free. If None,
|
||||
frees the maximum size of the ring buffer.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert self.is_writer, "Only the writer can free buffers."
|
||||
logger.debug(
|
||||
"Freeing up space in the ring buffer, "
|
||||
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
|
||||
|
||||
|
||||
class SingleWriterShmObjectStorage:
|
||||
"""
|
||||
A single-writer, multiple-reader object storage system built on top of a
|
||||
"""A single-writer, multiple-reader object storage system built on top of a
|
||||
shared memory ring buffer. Provides key-value storage with automatic memory
|
||||
management and cross-process serialization support.
|
||||
|
||||
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
|
||||
serde_class: type[ObjectSerde] = MsgpackSerde,
|
||||
reader_lock: LockType | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize the object storage.
|
||||
"""Initialize the object storage.
|
||||
|
||||
Args:
|
||||
max_object_size: Maximum size for a single object in bytes.
|
||||
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
|
||||
ring_buffer: The shared memory ring buffer for storing objects.
|
||||
serde_class: Serializer/deserializer for objects.
|
||||
reader_lock: Optional lock for synchronizing reader access.
|
||||
|
||||
Raises:
|
||||
ValueError: If reader_lock is None for readers.
|
||||
"""
|
||||
|
||||
"""
|
||||
self.max_object_size = max_object_size
|
||||
self.n_readers = n_readers
|
||||
self.serde_class = serde_class
|
||||
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
|
||||
del self.writer_flag[freed_id]
|
||||
|
||||
def is_cached(self, key: str) -> bool:
|
||||
"""
|
||||
Check if the object with the given key is cached.
|
||||
"""
|
||||
"""Check if the object with the given key is cached."""
|
||||
return key in self.key_index
|
||||
|
||||
def get_cached(self, key: str) -> tuple[int, int]:
|
||||
"""
|
||||
Get the cached object by key if it exists.
|
||||
"""
|
||||
"""Get the cached object by key if it exists."""
|
||||
address, monotonic_id = self.key_index[key]
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
return address, monotonic_id
|
||||
|
||||
def put(self, key: str, value: Any) -> tuple[int, int]:
|
||||
"""
|
||||
Store a key-value pair in the object storage.
|
||||
"""Store a key-value pair in the object storage.
|
||||
Attempts to free max_object_size bytes using FIFO order
|
||||
when the ring buffer runs out of space during a put() operation.
|
||||
|
||||
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
|
||||
MemoryError: If there's not enough space in the buffer
|
||||
ValueError: If the serialized object is too large
|
||||
ValueError: If the key already exists in the storage
|
||||
|
||||
"""
|
||||
if key in self.key_index:
|
||||
raise ValueError(f"Key '{key}' already exists in the storage.")
|
||||
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
|
||||
address: int = 0,
|
||||
monotonic_id: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Touch an existing cached item to update its eviction status.
|
||||
"""Touch an existing cached item to update its eviction status.
|
||||
|
||||
For writers (ShmObjectStoreSenderCache): Increment writer_flag
|
||||
For readers (ShmObjectStoreReceiverCache): Increment reader_count
|
||||
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
|
||||
)
|
||||
|
||||
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
|
||||
"""
|
||||
Default is_free function that checks if the first 4 bytes are zero.
|
||||
"""Default is_free function that checks if the first 4 bytes are zero.
|
||||
This indicates that the buffer is free.
|
||||
"""
|
||||
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
|
||||
|
||||
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ECConnectorBase Class for Distributed Encoder Cache &
|
||||
"""ECConnectorBase Class for Distributed Encoder Cache &
|
||||
P2P Encoder cache communication in V1
|
||||
|
||||
The class provides the following primitives:
|
||||
@@ -50,8 +49,7 @@ class ECConnectorRole(enum.Enum):
|
||||
|
||||
|
||||
class ECConnectorMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
"""Abstract Metadata used to communicate between the
|
||||
Scheduler ECConnector and Worker ECConnector.
|
||||
"""
|
||||
|
||||
@@ -59,8 +57,7 @@ class ECConnectorMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class ECConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
"""Abstract Metadata used to communicate back
|
||||
Worker ECConnector -> Scheduler ECConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
@@ -73,9 +70,7 @@ class ECConnectorWorkerMetadata(ABC):
|
||||
def aggregate(
|
||||
self, other: "ECConnectorWorkerMetadata"
|
||||
) -> "ECConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
|
||||
"""
|
||||
"""Aggregate metadata with another `ECConnectorWorkerMetadata` object."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -103,8 +98,7 @@ class ECConnectorBase(ABC):
|
||||
return self._is_consumer
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Shutdown the connector. This is called when the process
|
||||
"""Shutdown the connector. This is called when the process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
@@ -123,6 +117,7 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Args:
|
||||
connector_metadata (dict): the connector metadata.
|
||||
|
||||
"""
|
||||
self._connector_metadata = connector_metadata
|
||||
|
||||
@@ -141,8 +136,8 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
return self._connector_metadata
|
||||
@@ -151,10 +146,11 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
ec_caches: dict[str, torch.Tensor],
|
||||
):
|
||||
"""
|
||||
Initialize with the EC caches.
|
||||
"""Initialize with the EC caches.
|
||||
|
||||
Args:
|
||||
ec_caches: dictionary of encoder cache
|
||||
|
||||
"""
|
||||
# TODO: Implement this later for P2P feature
|
||||
return
|
||||
@@ -163,8 +159,7 @@ class ECConnectorBase(ABC):
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -174,6 +169,7 @@ class ECConnectorBase(ABC):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -181,8 +177,7 @@ class ECConnectorBase(ABC):
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -192,14 +187,14 @@ class ECConnectorBase(ABC):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -210,16 +205,17 @@ class ECConnectorBase(ABC):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the ECConnector worker metadata for this engine step.
|
||||
"""Build the ECConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
ECConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -232,8 +228,7 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a single encoder cache exists
|
||||
"""Check if a single encoder cache exists.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
@@ -241,14 +236,14 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
A bool where value is True if cache exist for
|
||||
the media
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""
|
||||
Ensure encoder cache items are available for the given request.
|
||||
"""Ensure encoder cache items are available for the given request.
|
||||
May initiate asynchronous transfers for items not yet local.
|
||||
|
||||
Args:
|
||||
@@ -258,6 +253,7 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
True if all items are ready or no transfer is needed.
|
||||
False if any items are still in transit (request should be deferred).
|
||||
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -282,11 +278,12 @@ class ECConnectorBase(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
Update ECConnector state to decide allocate cache for requests
|
||||
"""Update ECConnector state to decide allocate cache for requests.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
index (int): index of the encoder cache entry being allocated.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -294,37 +291,37 @@ class ECConnectorBase(ABC):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput):
|
||||
"""
|
||||
Update ECConnector state from worker-side connectors output.
|
||||
"""Update ECConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (ECConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its encoder cache is freed.
|
||||
"""Called when a request has finished, before its encoder cache is freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and cached
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
|
||||
"""
|
||||
return False, None
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
|
||||
@@ -229,6 +229,7 @@ class ECCPUScheduler:
|
||||
Returns:
|
||||
True if no item needs to wait for a remote encoding. False if any
|
||||
item is pending or the request must be aborted.
|
||||
|
||||
"""
|
||||
if not request.ec_transfer_params:
|
||||
return True
|
||||
@@ -259,6 +260,7 @@ class ECCPUScheduler:
|
||||
Raises:
|
||||
_RemoteUnavailable: The remote source is invalid, the read failed,
|
||||
or the item's total wait budget expired.
|
||||
|
||||
"""
|
||||
pos = feature.mm_position
|
||||
mm_hash = feature.identifier
|
||||
@@ -334,6 +336,7 @@ class ECCPUScheduler:
|
||||
item's remote announcement so subsequent steps do not retry it.
|
||||
False if only placeholder metadata (or no input) is available,
|
||||
recording the request for take_unavailable_requests() to drain.
|
||||
|
||||
"""
|
||||
mm_hash = feature.identifier
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
|
||||
@@ -15,7 +15,8 @@ from collections import OrderedDict
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
mutations only through EmbeddingCache methods.
|
||||
"""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
|
||||
@@ -67,8 +67,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
|
||||
|
||||
def start_load_caches(self, encoder_cache, **kwargs) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -78,6 +77,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -108,8 +108,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
|
||||
|
||||
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -119,6 +118,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
# Return if it is PD Instance
|
||||
if not self.is_producer:
|
||||
@@ -134,14 +134,14 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if cache exist externally for the media
|
||||
"""Check if cache exist externally for the media.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
|
||||
Returns:
|
||||
Bool indicate that media exists in cache or not
|
||||
|
||||
"""
|
||||
return self._found_match_for_mm_data(identifier)
|
||||
|
||||
@@ -150,9 +150,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
request: "Request",
|
||||
index: int,
|
||||
) -> None:
|
||||
"""
|
||||
Update ECConnector state after encoder cache allocation.
|
||||
"""
|
||||
"""Update ECConnector state after encoder cache allocation."""
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
# Only load cache if it is consumer and cache exists
|
||||
if not self.is_consumer or not self.has_cache_item(mm_hash):
|
||||
@@ -212,8 +210,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
mm_hash: str,
|
||||
create_folder: bool = True, # <- now defaults to True
|
||||
) -> str:
|
||||
"""
|
||||
Return the folder in which the cache for this mm_hash lives.
|
||||
"""Return the folder in which the cache for this mm_hash lives.
|
||||
If `create_folder` is True (default) the directory is created
|
||||
recursively the first time it is needed.
|
||||
"""
|
||||
@@ -223,8 +220,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(self, mm_hash: str) -> str:
|
||||
"""
|
||||
Return the full path of the safetensors file for this mm_hash.
|
||||
"""Return the full path of the safetensors file for this mm_hash.
|
||||
Ensures the parent directory exists because
|
||||
`_generate_foldername_debug` is called with its default
|
||||
(`create_folder=True`).
|
||||
|
||||
@@ -24,10 +24,7 @@ def has_ec_transfer() -> bool:
|
||||
|
||||
|
||||
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Initialize EC cache connector.
|
||||
"""
|
||||
|
||||
"""Initialize EC cache connector."""
|
||||
global _EC_CONNECTOR_AGENT
|
||||
|
||||
if vllm_config.ec_transfer_config is None:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
The async worker that transfers experts in the background.
|
||||
"""
|
||||
"""The async worker that transfers experts in the background."""
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user