Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+9 -12
View File
@@ -73,6 +73,7 @@ def make_copy_and_call(
Returns:
A wrapper function that copies inputs and calls the compiled function
"""
def copy_and_call(*args: Any) -> Any:
@@ -122,8 +123,7 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
class CompilerManager:
"""
A manager to manage the compilation process, including
"""A manager to manage the compilation process, including
caching the compiled graph, loading the compiled graph,
and compiling the graph.
@@ -150,7 +150,8 @@ class CompilerManager:
def compile_context(self, compile_range: Range) -> Generator[None, None, None]:
"""Provide compilation context for the duration of compilation to set
any torch global properties we want to scope to a single Inductor
compilation (e.g. partition rules, pass context)."""
compilation (e.g. partition rules, pass context).
"""
with pass_context(compile_range):
if self.compilation_config.use_inductor_graph_partition:
with inductor_partition_rule_context(
@@ -163,8 +164,7 @@ class CompilerManager:
def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
"""
Initialize the cache directory for the compiler.
"""Initialize the cache directory for the compiler.
The organization of the cache directory is as follows:
cache_dir=/path/to/hash_str/rank_i_j/prefix/
@@ -177,7 +177,6 @@ class CompilerManager:
base cache dir of /path/to/hash_str/rank_i_j/ ,
to store some common compilation artifacts.
"""
self.disable_cache = disable_cache
self.cache_dir = cache_dir
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
@@ -438,12 +437,10 @@ def _merge_empty_only_subgraphs(
node_to_subgraph_id: dict[fx.Node, int],
split_op_graphs: list[int],
) -> None:
"""
Merge a partition that only contains an empty allocation op into the
"""Merge a partition that only contains an empty allocation op into the
previous partition. This avoids generating standalone empty submodules,
which can lead to empty cudagraph captures.
"""
nodes_by_subgraph_id: dict[int, list[fx.Node]] = defaultdict(list)
for node, subgraph_id in node_to_subgraph_id.items():
nodes_by_subgraph_id[subgraph_id].append(node)
@@ -637,8 +634,7 @@ def wrap_with_cudagraph_if_needed(
is_first_graph: bool,
is_last_graph: bool,
) -> Any:
"""
Wrap a piecewise backend with CUDA graph wrapper if needed.
"""Wrap a piecewise backend with CUDA graph wrapper if needed.
This function is shared between VllmBackend and
construct_serializable_fn_from_inductor_cache.
@@ -651,6 +647,7 @@ def wrap_with_cudagraph_if_needed(
Returns:
The wrapped backend if CUDA graphs are enabled, otherwise the original backend
"""
if (
not compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
@@ -883,8 +880,8 @@ class VllmBackend:
sym_shape_indices
- returns_tuple_map: dict mapping submod_name to
returns_tuple
"""
"""
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
return None, None, None
+6 -6
View File
@@ -8,8 +8,7 @@ from vllm.config import CUDAGraphMode, VllmConfig
class AbstractStaticGraphWrapper(Protocol):
"""
StaticGraphWrapper interface that allows platforms to wrap a callable
"""StaticGraphWrapper interface that allows platforms to wrap a callable
to be captured as a static graph.
"""
@@ -20,8 +19,7 @@ class AbstractStaticGraphWrapper(Protocol):
runtime_mode: CUDAGraphMode,
**kwargs: Any,
) -> None:
"""
Initializes the StaticGraphWrapper class with graph capturing and
"""Initializes the StaticGraphWrapper class with graph capturing and
execution-related configurations.
Args:
@@ -31,15 +29,16 @@ class AbstractStaticGraphWrapper(Protocol):
graph runtime. See CUDAGraphMode in vllm/config.py.
Note that only the subset enum `NONE`, `PIECEWISE` and `FULL`
are used as concrete runtime mode for cudagraph dispatching.
Keyword Args:
kwargs: Additional keyword arguments for platform-specific
configurations.
"""
raise NotImplementedError
def __call__(self, *args: Any, **kwargs: Any) -> Any:
"""
Executes the wrapped callable.
"""Executes the wrapped callable.
If the current runtime mode in the ForwardContext matches the runtime
mode of this instance, it replays the CUDAGraph or captures it using
@@ -53,5 +52,6 @@ class AbstractStaticGraphWrapper(Protocol):
Returns:
Any: Output of the executed callable.
"""
raise NotImplementedError
+4 -5
View File
@@ -172,8 +172,7 @@ def patch_pytree_map_over_slice():
class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
"""
A wrapper around a compiled function by vllm. It will forward the tensor
"""A wrapper around a compiled function by vllm. It will forward the tensor
inputs to the compiled function and return the result.
It also implements a serialization interface to support PyTorch's precompile
with custom backend, so that we can save and load the compiled function on
@@ -410,9 +409,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
@property
def co_name(self) -> Literal["VllmSerializableFunction"]:
"""
Used for depyf debugging.
"""
"""Used for depyf debugging."""
return "VllmSerializableFunction"
@@ -445,6 +442,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
If modifying the backend creation/wrapping logic, consider updating both.
Args:
fake_mode: FakeTensorMode used to unpickle the cached graph.
state: Deserialized state dict containing graph_module, example_inputs,
prefix, sym_tensor_indices, is_encoder, etc.
standalone_compile_artifacts: The StandaloneCompiledArtifacts containing
@@ -455,6 +453,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
Returns:
A VllmSerializableFunction that can be called directly.
"""
from vllm.compilation.backends import (
VllmBackend,
+2
View File
@@ -154,6 +154,7 @@ def generate_execution_code(
consts is a list of non-primitive constant objects referenced
by the generated code via __vllm_consts__. These objects are
kept alive for the lifetime of the compiled function.
"""
code, submod_names, consts = generate_execution_code_with_name(
split_gm, "execution_fn", with_submod=True
@@ -181,6 +182,7 @@ def compile_execution_fn(
Returns:
A callable that executes the stitching logic.
"""
trace_structured(
"artifact",
+11 -21
View File
@@ -25,9 +25,7 @@ logger = init_logger(__name__)
class CompilerInterface:
"""
The interface for a compiler that can be used by vLLM.
"""
"""The interface for a compiler that can be used by vLLM."""
# The name of the compiler, e.g. inductor.
# This is a class-level attribute.
@@ -36,8 +34,7 @@ class CompilerInterface:
def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
"""
when the vLLM process uses `cache_dir` as the cache directory,
"""When the vLLM process uses `cache_dir` as the cache directory,
the compiler should initialize itself with the cache directory,
e.g. by re-directing its own cache directory to a sub-directory.
@@ -52,8 +49,7 @@ class CompilerInterface:
pass
def compute_hash(self, vllm_config: VllmConfig) -> str:
"""
Gather all the relevant information from the vLLM config,
"""Gather all the relevant information from the vLLM config,
to compute a hash so that we can cache the compiled model.
See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash]
@@ -71,8 +67,7 @@ class CompilerInterface:
compile_range: Range,
key: str | None = None,
) -> tuple[Callable[..., Any] | None, Any | None]:
"""
Compile the graph with the given example inputs and compiler config,
"""Compile the graph with the given example inputs and compiler config,
with a range. The `compile_range` specifies the range of the inputs,
it could be concrete size (if compile_sizes is provided), e.g. [4, 4]
or a range [5, 8].
@@ -105,8 +100,7 @@ class CompilerInterface:
graph_index: int,
compile_range: Range,
) -> Callable[..., Any]:
"""
Load the compiled function from the handle.
"""Load the compiled function from the handle.
Raises an error if the handle is invalid.
The handle is the second return value of the `compile` function.
@@ -115,8 +109,7 @@ class CompilerInterface:
class AlwaysHitShapeEnv:
"""
Why do we need this class:
"""Why do we need this class:
For normal `torch.compile` usage, every compilation will have
one Dynamo bytecode compilation and one Inductor compilation.
@@ -160,7 +153,8 @@ def _get_vllm_functorch_config() -> dict[str, Any]:
"""Return the functorch config overrides that vLLM applies at compile time.
Used by both set_functorch_config() and get_inductor_factors() to ensure
the compile-time config and cache key are always consistent."""
the compile-time config and cache key are always consistent.
"""
cfg: dict[str, Any] = {}
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
cfg["bundled_autograd_cache"] = False
@@ -249,8 +243,7 @@ def _patch_standalone_compile_atomic_save() -> None:
class InductorStandaloneAdaptor(CompilerInterface):
"""
The adaptor for the Inductor compiler.
"""The adaptor for the Inductor compiler.
Requires PyTorch 2.8+.
This is not on by default yet, but we plan to turn it on by default for
PyTorch 2.8.
@@ -447,9 +440,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
class InductorAdaptor(CompilerInterface):
"""
The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7.
"""
"""The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7."""
name = "inductor"
@@ -727,8 +718,7 @@ class InductorAdaptor(CompilerInterface):
return compiled_graph
def metrics_context(self) -> contextlib.AbstractContextManager[Any]:
"""
This method returns the Dynamo metrics context (if it exists,
"""This method returns the Dynamo metrics context (if it exists,
otherwise a null context). It is used by various compile components.
Present in torch>=2.6, it's used inside FxGraphCache in
torch==2.6 (but not after). It might also be used in various other
+1 -1
View File
@@ -38,7 +38,7 @@ class CUDAGraphStat:
class CUDAGraphLogging:
"""Aggregate and log cudagraph metrics"""
"""Aggregate and log cudagraph metrics."""
COLUMN_HEADERS = [
"Unpadded Tokens",
+4 -10
View File
@@ -56,8 +56,7 @@ def should_torch_compile_mm_encoder(vllm_config: VllmConfig) -> bool:
def ignore_torch_compile(cls: type[_T]) -> type[_T]:
"""
A decorator to ignore support_torch_compile decorator
"""A decorator to ignore support_torch_compile decorator
on the class. This is useful when a parent class has
a support_torch_compile decorator, but we don't want to
compile the class `cls` that inherits the parent class.
@@ -76,9 +75,7 @@ def ignore_torch_compile(cls: type[_T]) -> type[_T]:
def _should_ignore_torch_compile(cls: type[_T]) -> bool:
"""
Check if the class should be ignored for torch.compile.
"""
"""Check if the class should be ignored for torch.compile."""
return getattr(cls, IGNORE_COMPILE_KEY, False)
@@ -125,8 +122,7 @@ def support_torch_compile(
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
) -> Callable[[type[_T]], type[_T]] | type[_T]:
"""
A decorator to add support for compiling the forward method of a class.
"""A decorator to add support for compiling the forward method of a class.
Usage 1: use directly as a decorator without arguments:
@@ -338,7 +334,6 @@ def _support_torch_compile(
is_encoder: bool = False,
) -> type[_T]:
"""Internal implementation of support_torch_compile decorator."""
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
# support decorating multiple times
return cls
@@ -736,8 +731,7 @@ def _support_torch_compile(
def maybe_use_cudagraph_partition_wrapper(
vllm_config: VllmConfig,
) -> Generator[None, None, None]:
"""
Context manager to set/unset customized cudagraph partition wrappers.
"""Context manager to set/unset customized cudagraph partition wrappers.
If we're using Inductor-based graph partitioning, we currently have the
whole `fx.Graph` before Inductor lowering and the piecewise
+2 -3
View File
@@ -12,12 +12,10 @@ logger = init_logger(__name__)
def should_split(node: torch.fx.Node, splitting_ops: list[str]) -> bool:
"""
Check if a node should be split for dynamo graph partition.
"""Check if a node should be split for dynamo graph partition.
It operates on dynamo graph, so the node.target can be anything.
We need to check and split only on OpOverload and OpOverloadPacket.
"""
if node.op != "call_function":
return False
@@ -50,6 +48,7 @@ def inductor_partition_rule_context(
Args:
splitting_ops: List of operator names to partition on.
"""
if not splitting_ops:
logger.debug("No partition ops provided; skipping rule registration.")
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
class ActivationQuantPattern(VllmPatternReplacement):
"""
Base class for Activation+Quant fusions.
"""Base class for Activation+Quant fusions.
Should not be used directly.
"""
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+Fp8StaticQuant Pattern
"""
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
def __init__(self) -> None:
super().__init__(kFp8StaticTensorSym)
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+Nvfp4Quant Pattern
"""
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
def __init__(self) -> None:
super().__init__(kNvfp4Dynamic)
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
class SiluMulBlockQuantPattern(ActivationQuantPattern):
"""
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
Supports group_size 128 and 64 via QuantKey.
Parameterized on is_scale_transposed for different scale layouts.
"""
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses a pre-defined set of custom ops into fused ops.
"""This pass fuses a pre-defined set of custom ops into fused ops.
It uses the torch pattern matcher to find the patterns and replace them.
Because patterns can only be registered once, the pass is a singleton.
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
"""
for node in match.nodes:
if node.target == _IR_RMS_NORM_OP:
x, weight = node.args[0], node.args[1]
@@ -347,8 +348,7 @@ class BasePattern:
class AllReduceRMSNormPattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
with fused flashinfer implementation.
Applies to allreduce + rmsnorm before attn in the first Transformer block.
"""
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
class AllReduceFusedAddRMSNormPattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
"""
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
+ static fp8 quant with fused flashinfer implementation.
Applies to allreduce + rmsnorm + quant before attn
in the first Transformer block.
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
+ static fp8 quant with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn + quant and
mlp + rmsnorm + quant before attn.
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (without residual)
"""This pattern replaces the allreduce + rms norm (without residual)
+ static nvfp4 quant with fused flashinfer implementation.
Applies to allreduce + rmsnorm + quant before attn
in the first Transformer block.
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
"""
This pattern replaces the allreduce + rms norm (with residual)
"""This pattern replaces the allreduce + rms norm (with residual)
+ static nvfp4 quant with fused flashinfer implementation.
Applies to o_proj + rmsnorm after attn + quant and
mlp + rmsnorm + quant before attn.
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""
Fusion for Attention+Fp8StaticQuant.
"""Fusion for Attention+Fp8StaticQuant.
Only triggers when the attention implementation returns True in
`fused_output_quant_supported()`. If the pattern is found, the
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
class AttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for Attention+Nvfp4Quant.
"""Fusion for Attention+Nvfp4Quant.
Only triggers when the attention implementation returns True in
`fused_output_quant_supported()`. If the pattern is found, the
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses post-attention quantization onto attention if supported.
"""This pass fuses post-attention quantization onto attention if supported.
It uses the pattern matcher and matches each layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
def inputs(self) -> list[torch.Tensor]:
"""Utility for inputs to the pattern"""
"""Utility for inputs to the pattern."""
raise NotImplementedError
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""
Fusion for MLA Attention+Fp8StaticQuant.
"""Fusion for MLA Attention+Fp8StaticQuant.
Matches the pattern: MLA attention -> static FP8 quant, and replaces
it with MLA attention(output_scale=scale, output=fp8_buffer).
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
class MLAAttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for MLA Attention+Nvfp4Quant.
"""Fusion for MLA Attention+Nvfp4Quant.
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
class MLAAttnFp8GroupQuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
replaces it with MLA attention(output_block_scale=group_scale_buffer).
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses post-attention quantization onto MLA attention if supported.
"""This pass fuses post-attention quantization onto MLA attention if supported.
It uses the pattern matcher and matches each MLA layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
@@ -34,8 +34,7 @@ P = ParamSpec("P")
class QkNormRopePattern:
"""
Match the unfused sequence in attention blocks and replace with the fused op.
"""Match the unfused sequence in attention blocks and replace with the fused op.
Unfused (conceptually):
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
@@ -106,8 +106,7 @@ direct_register_custom_op(
class QkNormRopeKvCachePattern:
"""
Match the unfused sequence:
"""Match the unfused sequence:
q, k, v = split(qkv, ...)
q = rms_norm(q.view(heads), q_weight).view(flat)
k = rms_norm(k.view(heads), k_weight).view(flat)
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
"""
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
attention layers that support the combined operation, eliminating two
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
class FusedRMSQuantKey(NamedTuple):
"""
Named tuple for identifying the type of RMSNorm + quant fusion.
"""Named tuple for identifying the type of RMSNorm + quant fusion.
quant: type of quantization
fused_add: does the op also perform the residual add
"""
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
It also supports fused_add_rms_norm.
"""
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
This pattern fuses aiter rms_norm & group fp8 quant custom
"""This pattern fuses aiter rms_norm & group fp8 quant custom
ops into an aiter rms_norm_group_fp8_quant op.
"""
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
into a aiter rms_norm_with_add_group_fp8_quant op.
"""
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Pattern matching ``rms_norm`` whose output feeds *two* distinct
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
independent fused ``rms_norm_group_fp8_quant`` ops.
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
"""
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
with rocm_aiter_fused_rms_gated_fp8_group_quant.
The norm operates per-head on (N*H, D) tensors. The compiler folds the
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
into a fused rms_norm_quant op.
It also supports fused_add_rms_norm.
"""
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
"""
This pattern fuses aiter silu_and_mul & group fp8 quant custom
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
ops into an aiter silu_and_mul_group_fp8_quant op.
"""
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
"""
This pass fuses a pre-defined set of custom ops into fused ops.
"""This pass fuses a pre-defined set of custom ops into fused ops.
It uses the torch pattern matcher to find the patterns and replace them.
Because patterns can only be registered once, the pass is a singleton.
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
class AddAiterRMSNormPadPattern:
"""
This pattern replaces an aiter_rmsnorm_with_add & a pad op
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
with a custom triton_add_rmsnorm_pad op from AITER.
"""
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
"""
This pass replaces an AITER CK RMSNorm + residual add and a pad op
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
with an triton_add_rmsnorm_pad op from AITER.
"""
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
class MLADualRMSNormPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
):
"""
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
AITER's ``fused_qk_rmsnorm`` HIP kernel.
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
],
]
):
"""
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
is_neox: bool,
layer_name: LayerNameType,
) -> torch.Tensor:
"""
This impl fetches the KV cache and slot mapping from the forward context,
"""This impl fetches the KV cache and slot mapping from the forward context,
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
that is passed to unified_attention to signal a side effect and
@@ -98,8 +97,7 @@ direct_register_custom_op(
class RopeStaticQQuantKVCachePattern:
"""
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
dependency ordering.
"""
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
class RopeReshapeKVCachePattern:
"""
This pattern matches the following unfused inplace ops:
"""This pattern matches the following unfused inplace ops:
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
"""
This pass fuses the rotary embedding and KV cache update operations
"""This pass fuses the rotary embedding and KV cache update operations
into a single fused kernel if available.
It uses the pattern matcher and matches each layer manually, as strings
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
tp_size: int,
element_size: int,
) -> int | None:
"""
Calculate the minimum token threshold for applying sequence parallelism.
"""Calculate the minimum token threshold for applying sequence parallelism.
Returns None if sequence parallelism should not be applied based on model size.
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
class SequenceParallelismPass(VllmPatternMatcherPass):
"""
This pass enables sequence parallelism for models.
"""This pass enables sequence parallelism for models.
It identifies patterns where an AllReduce operation is followed by
an RMSNorm (or RMSNorm and then Quantization) operation.
These patterns are replaced with a ReduceScatter operation, followed by
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
self.dump_patterns(config, self.patterns)
def is_applicable_for_range(self, compile_range: Range) -> bool:
"""
Determines if sequence parallelism should be applied for the given
"""Determines if sequence parallelism should be applied for the given
compile range.
SP is only beneficial for larger batch sizes where the communication
+8 -12
View File
@@ -64,14 +64,12 @@ def _hash_source_cached(*srcs: str | type | types.FunctionType) -> str:
class InductorPass(CustomGraphPass): # type: ignore[misc]
"""
A custom graph pass that uses a hash of its source as the UUID.
"""A custom graph pass that uses a hash of its source as the UUID.
This is defined as a convenience and should work in most cases.
"""
def uuid(self) -> str:
"""
Provide a unique identifier for the pass, used in Inductor code cache.
"""Provide a unique identifier for the pass, used in Inductor code cache.
This should depend on the pass implementation, so that changes to the
pass result in recompilation.
By default, the object source is hashed.
@@ -80,14 +78,14 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
@staticmethod
def hash_source(*srcs: str | Any) -> str:
"""
Utility method to hash the sources of functions or objects.
"""Utility method to hash the sources of functions or objects.
Args:
srcs: strings or objects to add to the hash.
Objects and functions have their source inspected.
Results are cached by resolved types to avoid repeated
inspect.getsource() calls.
"""
# Resolve instances to their class for a hashable cache key.
cache_key = tuple(
@@ -98,11 +96,11 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
@staticmethod
def hash_dict(dict_: dict[Any, Any]) -> str:
"""
Utility method to hash a dictionary, can alternatively be used for uuid.
"""Utility method to hash a dictionary, can alternatively be used for uuid.
Returns:
A sha256 hash of the json rep of the dictionary.
"""
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
@@ -112,8 +110,7 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
class CallableInductorPass(InductorPass):
"""
This class is a wrapper for a callable that automatically provides an
"""This class is a wrapper for a callable that automatically provides an
implementation of the UUID.
"""
@@ -131,8 +128,7 @@ class CallableInductorPass(InductorPass):
def enable_fake_mode(fn: Callable[P, R]) -> Callable[P, R]:
"""
Applies a FakeTensorMode context. This is useful when you don't want to
"""Applies a FakeTensorMode context. This is useful when you don't want to
create or run things with real tensors.
"""
@@ -70,8 +70,7 @@ def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
class UnsafeCloneEliminationPass(VllmInductorPass):
"""
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
"""This pass removes clone nodes that are no longer needed after vLLM IR lowering.
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
contents of non-donated graph inputs.
@@ -19,8 +19,7 @@ logger = init_logger(__name__)
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
"""
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
"""This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
The maybe_inplace overloads have the same signature as the default overload
so the pass simply replaces the called overload.
That makes the graph properly functional.
+2 -5
View File
@@ -23,9 +23,7 @@ logger = init_logger(__name__)
class VllmIRLoweringPass(VllmInductorPass):
"""
This pass lowers vLLM IR ops to their implementations the priority list.
"""
"""This pass lowers vLLM IR ops to their implementations the priority list."""
def __init__(self, vllm_config: VllmConfig) -> None:
super().__init__(vllm_config)
@@ -113,8 +111,7 @@ class VllmIRLoweringPass(VllmInductorPass):
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
def uuid(self) -> str:
"""
IR op priority & impl sources affect lowering pass output,
"""IR op priority & impl sources affect lowering pass output,
so we include them in the cache key.
"""
priorities = {name: op.get_priority() for name, op in IrOp.registry.items()}
+3 -6
View File
@@ -71,8 +71,7 @@ R = TypeVar("R")
def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
"""
Function decorator that turns on inductor pattern match debug
"""Function decorator that turns on inductor pattern match debug
for the duration of the call.
Used to avoid logging builtin Inductor pattern matching.
"""
@@ -89,8 +88,7 @@ def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
"""
The pass manager for post-grad passes.
"""The pass manager for post-grad passes.
It handles configuration, adding custom passes, and running passes.
It supports uuid for the Inductor code cache. That includes torch<2.6
support using pickling (in .inductor_pass.CustomGraphPass).
@@ -236,8 +234,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes.append(pass_)
def uuid(self) -> str:
"""
The PostGradPassManager is set as a custom pass in the Inductor and
"""The PostGradPassManager is set as a custom pass in the Inductor and
affects compilation caching. Its uuid depends on the UUIDs of all
dependent passes and the pass config. See InductorPass for more info.
"""
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
class FixFunctionalizationPass(VllmInductorPass):
"""
This pass defunctionalizes certain nodes to avoid redundant tensor copies.
"""This pass defunctionalizes certain nodes to avoid redundant tensor copies.
After this pass, DCE (dead-code elimination) should never be run,
as de-functionalized nodes may appear as dead code.
@@ -260,9 +259,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self.nodes_to_remove.clear()
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]) -> None:
"""
Stage a node (or nodes) for removal at the end of the pass.
"""
"""Stage a node (or nodes) for removal at the end of the pass."""
if isinstance(node_or_nodes, torch.fx.Node):
self.nodes_to_remove.append(node_or_nodes)
else:
@@ -275,8 +272,7 @@ class FixFunctionalizationPass(VllmInductorPass):
mutated_args: dict[int, torch.fx.Node | str],
args: tuple[torch.fx.Node | str, ...] | None = None,
) -> None:
"""
De-functionalize a node by replacing it with a call to the original.
"""De-functionalize a node by replacing it with a call to the original.
It also replaces the getitem users with the mutated arguments.
See replace_users_with_mutated_args and insert_defunctionalized.
"""
@@ -287,14 +283,14 @@ class FixFunctionalizationPass(VllmInductorPass):
def replace_users_with_mutated_args(
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
) -> None:
"""
Replace mutated getitem users of the auto-functionalized node with the
"""Replace mutated getitem users of the auto-functionalized node with the
mutated arguments.
Args:
node: The auto-functionalized node
mutated_args: The mutated arguments, indexed by getitem index.
If the value of an arg is a string, `node.kwargs[arg]` is used.
"""
for idx, user in self.getitem_users(node).items():
# Some functionalized nodes may return both a result at getitem[0]
@@ -310,8 +306,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self._remove(user)
def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]:
"""
Returns the operator.getitem users of the auto-functionalized node,
"""Returns the operator.getitem users of the auto-functionalized node,
indexed by the index they are getting.
"""
users = {}
@@ -327,8 +322,7 @@ class FixFunctionalizationPass(VllmInductorPass):
node: torch.fx.Node,
args: tuple[torch.fx.Node | str, ...] | None = None,
) -> None:
"""
Insert a new defunctionalized node into the graph before node.
"""Insert a new defunctionalized node into the graph before node.
If one of the kwargs is 'out', provide args directly,
as node.kwargs cannot be used.
See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
@@ -338,6 +332,7 @@ class FixFunctionalizationPass(VllmInductorPass):
node: The auto-functionalized node to defunctionalize
args: If we cannot use kwargs, specify args directly.
If an arg is a string, `node.kwargs[arg]` is used.
""" # noqa: E501
assert is_func(node, auto_functionalized), (
f"node must be auto-functionalized, is {node} instead"
@@ -16,8 +16,7 @@ logger = init_logger(__name__)
class NoOpEliminationPass(VllmInductorPass):
"""
This is an inductor pass that removes redundant reshape/slice operations.
"""This is an inductor pass that removes redundant reshape/slice operations.
It is required for RMSNorm-quant fusion to work properly.
That's because apply_fp8_linear adds a reshape, which is redundant
in the 2D-case. Additionally, torch internal no-op elimination pass does
@@ -106,8 +105,7 @@ class NoOpEliminationPass(VllmInductorPass):
# ---------------------- Shape comparison helpers ----------------------
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
"""
This function checks if two dimensions are equivalent.
"""This function checks if two dimensions are equivalent.
Args:
dim: The dimension arg to reshape/slice
@@ -119,6 +117,7 @@ class NoOpEliminationPass(VllmInductorPass):
There are two cases in which the dimensions are equivalent:
1. The dimensions are equal (both integers)
2. The dimensions both correspond to the same SymInt
"""
# Case 1
return statically_known_true(dim == i_dim) # type: ignore[no-any-return]
@@ -6,8 +6,7 @@ from ..vllm_inductor_pass import VllmInductorPass
class PostCleanupPass(VllmInductorPass):
"""
This pass performs cleanup after custom passes.
"""This pass performs cleanup after custom passes.
It topologically sorts the graph and removes unused nodes.
This is needed because the pattern matcher does not guarantee producing
a topologically sorted graph, and there may be unused nodes left around.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
"""Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
assignment if there are no users for the inplace tensor written to by
the slice_scatter call.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
input tensor with the same split sizes.
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
@@ -10,9 +9,10 @@ that CSE fails to merge. This pass detects and replaces the duplicates
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
see a single split node with all users attached.
See also:
See Also:
- vLLM #33295 (original issue)
- PyTorch #174472 (upstream CSE gap)
"""
import operator
@@ -30,7 +30,8 @@ logger = init_logger(__name__)
class SplitCoalescingPass(VllmInductorPass):
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
node when they share the same input tensor and split sizes."""
node when they share the same input tensor and split sizes.
"""
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
+7 -13
View File
@@ -35,8 +35,7 @@ class InductorCompilationConfig:
class VllmInductorPass(InductorPass):
"""
An inductor pass with access to vLLM PassConfig.
"""An inductor pass with access to vLLM PassConfig.
It provides timing, logging, and dumping utilities.
"""
@@ -93,8 +92,7 @@ def get_match_table() -> dict[str, int]:
class VllmPatternMatcherPass(VllmInductorPass):
"""
A VllmInductorPass that uses the Inductor pattern matcher.
"""A VllmInductorPass that uses the Inductor pattern matcher.
Provides pattern registration with match counting, debug dumping, and logging.
"""
@@ -109,7 +107,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
)
def _replace_op_overloads(self, string: str) -> str:
"""Replace <OpOverload(..., ...)> with nicer formulations"""
"""Replace <OpOverload(..., ...)> with nicer formulations."""
return str(
self._OP_OVERLOAD_PATTERN.sub(
lambda m: f"torch.ops.{m.group(1)}.{m.group(2)}",
@@ -123,8 +121,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
logger.debug("fusion pass matches: %s", dict(cls.match_table))
def dump_patterns(self, config: VllmConfig, pm_pass: PatternMatcherPass) -> None:
"""
If debug dumping is enabled, dump the Inductor pattern-matcher patterns
"""If debug dumping is enabled, dump the Inductor pattern-matcher patterns
into the debug_dump_path folder next to the dumped fx graphs.
This method does its best to print something that looks like Python code
@@ -195,8 +192,7 @@ R = TypeVar("R")
class VllmPatternReplacement(ABC, Generic[P, R]):
"""
A pattern/replacement pair for FX graph fusion.
"""A pattern/replacement pair for FX graph fusion.
Implement the three abstract members below, then pass
instances to VllmFusionPatternMatcherPass.register(). The pass will
@@ -216,8 +212,7 @@ class VllmPatternReplacement(ABC, Generic[P, R]):
@property
@abstractmethod
def replacement(self) -> Callable[P, R]:
"""
Returns a closure defining the FX subgraph to
"""Returns a closure defining the FX subgraph to
substitute in place of each match.
"""
...
@@ -294,8 +289,7 @@ def _remove_noop_permutes(gm: fx.GraphModule) -> None:
class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):
"""
A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
"""A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
Subclasses register patterns via self.register() in their own __init__.
"""
+1 -2
View File
@@ -96,8 +96,7 @@ class PiecewiseBackend:
compiled_runnables: dict[str, Callable[..., Any]] | None = None,
submod_name: str = "",
):
"""
The backend for piecewise compilation.
"""The backend for piecewise compilation.
It mainly handles the compilation of static shapes and
dispatching based on runtime shape.
+3 -7
View File
@@ -45,8 +45,7 @@ def _compilation_context() -> Generator[None, None, None]:
class TorchCompileWithNoGuardsWrapper:
"""
A wrapper class for torch.compile, it ensures that all guards are dropped
"""A wrapper class for torch.compile, it ensures that all guards are dropped
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
When guards are dropped, the first time __call__ is invoked, a single
compilation is triggered. Dynamo should never be traced again after that
@@ -272,8 +271,7 @@ class TorchCompileWithNoGuardsWrapper:
@contextmanager
def _dispatch_to_compiled_code(self) -> Generator[None, None, None]:
# noqa: E501
"""
Context manager to dispatch to internally compiled code for torch<2.8.
"""Context manager to dispatch to internally compiled code for torch<2.8.
Why does this work? Because Dynamo guarantees that the compiled
bytecode has exactly the same arguments, cell variables, and free
variables as the original code. Therefore we can directly switch
@@ -291,9 +289,7 @@ class TorchCompileWithNoGuardsWrapper:
def reset_compile_wrapper(model: torch.nn.Module) -> None:
"""
Clean up compiled model and captured CUDA graphs for elastic EP.
"""
"""Clean up compiled model and captured CUDA graphs for elastic EP."""
if not isinstance(model, TorchCompileWithNoGuardsWrapper) and hasattr(
model, "model"
):