Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -73,6 +73,7 @@ def make_copy_and_call(
|
||||
|
||||
Returns:
|
||||
A wrapper function that copies inputs and calls the compiled function
|
||||
|
||||
"""
|
||||
|
||||
def copy_and_call(*args: Any) -> Any:
|
||||
@@ -122,8 +123,7 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
|
||||
|
||||
|
||||
class CompilerManager:
|
||||
"""
|
||||
A manager to manage the compilation process, including
|
||||
"""A manager to manage the compilation process, including
|
||||
caching the compiled graph, loading the compiled graph,
|
||||
and compiling the graph.
|
||||
|
||||
@@ -150,7 +150,8 @@ class CompilerManager:
|
||||
def compile_context(self, compile_range: Range) -> Generator[None, None, None]:
|
||||
"""Provide compilation context for the duration of compilation to set
|
||||
any torch global properties we want to scope to a single Inductor
|
||||
compilation (e.g. partition rules, pass context)."""
|
||||
compilation (e.g. partition rules, pass context).
|
||||
"""
|
||||
with pass_context(compile_range):
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
with inductor_partition_rule_context(
|
||||
@@ -163,8 +164,7 @@ class CompilerManager:
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the cache directory for the compiler.
|
||||
"""Initialize the cache directory for the compiler.
|
||||
|
||||
The organization of the cache directory is as follows:
|
||||
cache_dir=/path/to/hash_str/rank_i_j/prefix/
|
||||
@@ -177,7 +177,6 @@ class CompilerManager:
|
||||
base cache dir of /path/to/hash_str/rank_i_j/ ,
|
||||
to store some common compilation artifacts.
|
||||
"""
|
||||
|
||||
self.disable_cache = disable_cache
|
||||
self.cache_dir = cache_dir
|
||||
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
|
||||
@@ -438,12 +437,10 @@ def _merge_empty_only_subgraphs(
|
||||
node_to_subgraph_id: dict[fx.Node, int],
|
||||
split_op_graphs: list[int],
|
||||
) -> None:
|
||||
"""
|
||||
Merge a partition that only contains an empty allocation op into the
|
||||
"""Merge a partition that only contains an empty allocation op into the
|
||||
previous partition. This avoids generating standalone empty submodules,
|
||||
which can lead to empty cudagraph captures.
|
||||
"""
|
||||
|
||||
nodes_by_subgraph_id: dict[int, list[fx.Node]] = defaultdict(list)
|
||||
for node, subgraph_id in node_to_subgraph_id.items():
|
||||
nodes_by_subgraph_id[subgraph_id].append(node)
|
||||
@@ -637,8 +634,7 @@ def wrap_with_cudagraph_if_needed(
|
||||
is_first_graph: bool,
|
||||
is_last_graph: bool,
|
||||
) -> Any:
|
||||
"""
|
||||
Wrap a piecewise backend with CUDA graph wrapper if needed.
|
||||
"""Wrap a piecewise backend with CUDA graph wrapper if needed.
|
||||
This function is shared between VllmBackend and
|
||||
construct_serializable_fn_from_inductor_cache.
|
||||
|
||||
@@ -651,6 +647,7 @@ def wrap_with_cudagraph_if_needed(
|
||||
|
||||
Returns:
|
||||
The wrapped backend if CUDA graphs are enabled, otherwise the original backend
|
||||
|
||||
"""
|
||||
if (
|
||||
not compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
|
||||
@@ -883,8 +880,8 @@ class VllmBackend:
|
||||
sym_shape_indices
|
||||
- returns_tuple_map: dict mapping submod_name to
|
||||
returns_tuple
|
||||
"""
|
||||
|
||||
"""
|
||||
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
|
||||
return None, None, None
|
||||
|
||||
|
||||
@@ -8,8 +8,7 @@ from vllm.config import CUDAGraphMode, VllmConfig
|
||||
|
||||
|
||||
class AbstractStaticGraphWrapper(Protocol):
|
||||
"""
|
||||
StaticGraphWrapper interface that allows platforms to wrap a callable
|
||||
"""StaticGraphWrapper interface that allows platforms to wrap a callable
|
||||
to be captured as a static graph.
|
||||
"""
|
||||
|
||||
@@ -20,8 +19,7 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
runtime_mode: CUDAGraphMode,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Initializes the StaticGraphWrapper class with graph capturing and
|
||||
"""Initializes the StaticGraphWrapper class with graph capturing and
|
||||
execution-related configurations.
|
||||
|
||||
Args:
|
||||
@@ -31,15 +29,16 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
graph runtime. See CUDAGraphMode in vllm/config.py.
|
||||
Note that only the subset enum `NONE`, `PIECEWISE` and `FULL`
|
||||
are used as concrete runtime mode for cudagraph dispatching.
|
||||
|
||||
Keyword Args:
|
||||
kwargs: Additional keyword arguments for platform-specific
|
||||
configurations.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
"""
|
||||
Executes the wrapped callable.
|
||||
"""Executes the wrapped callable.
|
||||
|
||||
If the current runtime mode in the ForwardContext matches the runtime
|
||||
mode of this instance, it replays the CUDAGraph or captures it using
|
||||
@@ -53,5 +52,6 @@ class AbstractStaticGraphWrapper(Protocol):
|
||||
|
||||
Returns:
|
||||
Any: Output of the executed callable.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -172,8 +172,7 @@ def patch_pytree_map_over_slice():
|
||||
|
||||
|
||||
class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
|
||||
"""
|
||||
A wrapper around a compiled function by vllm. It will forward the tensor
|
||||
"""A wrapper around a compiled function by vllm. It will forward the tensor
|
||||
inputs to the compiled function and return the result.
|
||||
It also implements a serialization interface to support PyTorch's precompile
|
||||
with custom backend, so that we can save and load the compiled function on
|
||||
@@ -410,9 +409,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc]
|
||||
|
||||
@property
|
||||
def co_name(self) -> Literal["VllmSerializableFunction"]:
|
||||
"""
|
||||
Used for depyf debugging.
|
||||
"""
|
||||
"""Used for depyf debugging."""
|
||||
return "VllmSerializableFunction"
|
||||
|
||||
|
||||
@@ -445,6 +442,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
|
||||
If modifying the backend creation/wrapping logic, consider updating both.
|
||||
|
||||
Args:
|
||||
fake_mode: FakeTensorMode used to unpickle the cached graph.
|
||||
state: Deserialized state dict containing graph_module, example_inputs,
|
||||
prefix, sym_tensor_indices, is_encoder, etc.
|
||||
standalone_compile_artifacts: The StandaloneCompiledArtifacts containing
|
||||
@@ -455,6 +453,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
|
||||
|
||||
Returns:
|
||||
A VllmSerializableFunction that can be called directly.
|
||||
|
||||
"""
|
||||
from vllm.compilation.backends import (
|
||||
VllmBackend,
|
||||
|
||||
@@ -154,6 +154,7 @@ def generate_execution_code(
|
||||
consts is a list of non-primitive constant objects referenced
|
||||
by the generated code via __vllm_consts__. These objects are
|
||||
kept alive for the lifetime of the compiled function.
|
||||
|
||||
"""
|
||||
code, submod_names, consts = generate_execution_code_with_name(
|
||||
split_gm, "execution_fn", with_submod=True
|
||||
@@ -181,6 +182,7 @@ def compile_execution_fn(
|
||||
|
||||
Returns:
|
||||
A callable that executes the stitching logic.
|
||||
|
||||
"""
|
||||
trace_structured(
|
||||
"artifact",
|
||||
|
||||
@@ -25,9 +25,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompilerInterface:
|
||||
"""
|
||||
The interface for a compiler that can be used by vLLM.
|
||||
"""
|
||||
"""The interface for a compiler that can be used by vLLM."""
|
||||
|
||||
# The name of the compiler, e.g. inductor.
|
||||
# This is a class-level attribute.
|
||||
@@ -36,8 +34,7 @@ class CompilerInterface:
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
) -> None:
|
||||
"""
|
||||
when the vLLM process uses `cache_dir` as the cache directory,
|
||||
"""When the vLLM process uses `cache_dir` as the cache directory,
|
||||
the compiler should initialize itself with the cache directory,
|
||||
e.g. by re-directing its own cache directory to a sub-directory.
|
||||
|
||||
@@ -52,8 +49,7 @@ class CompilerInterface:
|
||||
pass
|
||||
|
||||
def compute_hash(self, vllm_config: VllmConfig) -> str:
|
||||
"""
|
||||
Gather all the relevant information from the vLLM config,
|
||||
"""Gather all the relevant information from the vLLM config,
|
||||
to compute a hash so that we can cache the compiled model.
|
||||
|
||||
See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash]
|
||||
@@ -71,8 +67,7 @@ class CompilerInterface:
|
||||
compile_range: Range,
|
||||
key: str | None = None,
|
||||
) -> tuple[Callable[..., Any] | None, Any | None]:
|
||||
"""
|
||||
Compile the graph with the given example inputs and compiler config,
|
||||
"""Compile the graph with the given example inputs and compiler config,
|
||||
with a range. The `compile_range` specifies the range of the inputs,
|
||||
it could be concrete size (if compile_sizes is provided), e.g. [4, 4]
|
||||
or a range [5, 8].
|
||||
@@ -105,8 +100,7 @@ class CompilerInterface:
|
||||
graph_index: int,
|
||||
compile_range: Range,
|
||||
) -> Callable[..., Any]:
|
||||
"""
|
||||
Load the compiled function from the handle.
|
||||
"""Load the compiled function from the handle.
|
||||
Raises an error if the handle is invalid.
|
||||
|
||||
The handle is the second return value of the `compile` function.
|
||||
@@ -115,8 +109,7 @@ class CompilerInterface:
|
||||
|
||||
|
||||
class AlwaysHitShapeEnv:
|
||||
"""
|
||||
Why do we need this class:
|
||||
"""Why do we need this class:
|
||||
|
||||
For normal `torch.compile` usage, every compilation will have
|
||||
one Dynamo bytecode compilation and one Inductor compilation.
|
||||
@@ -160,7 +153,8 @@ def _get_vllm_functorch_config() -> dict[str, Any]:
|
||||
"""Return the functorch config overrides that vLLM applies at compile time.
|
||||
|
||||
Used by both set_functorch_config() and get_inductor_factors() to ensure
|
||||
the compile-time config and cache key are always consistent."""
|
||||
the compile-time config and cache key are always consistent.
|
||||
"""
|
||||
cfg: dict[str, Any] = {}
|
||||
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
|
||||
cfg["bundled_autograd_cache"] = False
|
||||
@@ -249,8 +243,7 @@ def _patch_standalone_compile_atomic_save() -> None:
|
||||
|
||||
|
||||
class InductorStandaloneAdaptor(CompilerInterface):
|
||||
"""
|
||||
The adaptor for the Inductor compiler.
|
||||
"""The adaptor for the Inductor compiler.
|
||||
Requires PyTorch 2.8+.
|
||||
This is not on by default yet, but we plan to turn it on by default for
|
||||
PyTorch 2.8.
|
||||
@@ -447,9 +440,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
|
||||
|
||||
|
||||
class InductorAdaptor(CompilerInterface):
|
||||
"""
|
||||
The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7.
|
||||
"""
|
||||
"""The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7."""
|
||||
|
||||
name = "inductor"
|
||||
|
||||
@@ -727,8 +718,7 @@ class InductorAdaptor(CompilerInterface):
|
||||
return compiled_graph
|
||||
|
||||
def metrics_context(self) -> contextlib.AbstractContextManager[Any]:
|
||||
"""
|
||||
This method returns the Dynamo metrics context (if it exists,
|
||||
"""This method returns the Dynamo metrics context (if it exists,
|
||||
otherwise a null context). It is used by various compile components.
|
||||
Present in torch>=2.6, it's used inside FxGraphCache in
|
||||
torch==2.6 (but not after). It might also be used in various other
|
||||
|
||||
@@ -38,7 +38,7 @@ class CUDAGraphStat:
|
||||
|
||||
|
||||
class CUDAGraphLogging:
|
||||
"""Aggregate and log cudagraph metrics"""
|
||||
"""Aggregate and log cudagraph metrics."""
|
||||
|
||||
COLUMN_HEADERS = [
|
||||
"Unpadded Tokens",
|
||||
|
||||
@@ -56,8 +56,7 @@ def should_torch_compile_mm_encoder(vllm_config: VllmConfig) -> bool:
|
||||
|
||||
|
||||
def ignore_torch_compile(cls: type[_T]) -> type[_T]:
|
||||
"""
|
||||
A decorator to ignore support_torch_compile decorator
|
||||
"""A decorator to ignore support_torch_compile decorator
|
||||
on the class. This is useful when a parent class has
|
||||
a support_torch_compile decorator, but we don't want to
|
||||
compile the class `cls` that inherits the parent class.
|
||||
@@ -76,9 +75,7 @@ def ignore_torch_compile(cls: type[_T]) -> type[_T]:
|
||||
|
||||
|
||||
def _should_ignore_torch_compile(cls: type[_T]) -> bool:
|
||||
"""
|
||||
Check if the class should be ignored for torch.compile.
|
||||
"""
|
||||
"""Check if the class should be ignored for torch.compile."""
|
||||
return getattr(cls, IGNORE_COMPILE_KEY, False)
|
||||
|
||||
|
||||
@@ -125,8 +122,7 @@ def support_torch_compile(
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
) -> Callable[[type[_T]], type[_T]] | type[_T]:
|
||||
"""
|
||||
A decorator to add support for compiling the forward method of a class.
|
||||
"""A decorator to add support for compiling the forward method of a class.
|
||||
|
||||
Usage 1: use directly as a decorator without arguments:
|
||||
|
||||
@@ -338,7 +334,6 @@ def _support_torch_compile(
|
||||
is_encoder: bool = False,
|
||||
) -> type[_T]:
|
||||
"""Internal implementation of support_torch_compile decorator."""
|
||||
|
||||
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
|
||||
# support decorating multiple times
|
||||
return cls
|
||||
@@ -736,8 +731,7 @@ def _support_torch_compile(
|
||||
def maybe_use_cudagraph_partition_wrapper(
|
||||
vllm_config: VllmConfig,
|
||||
) -> Generator[None, None, None]:
|
||||
"""
|
||||
Context manager to set/unset customized cudagraph partition wrappers.
|
||||
"""Context manager to set/unset customized cudagraph partition wrappers.
|
||||
|
||||
If we're using Inductor-based graph partitioning, we currently have the
|
||||
whole `fx.Graph` before Inductor lowering and the piecewise
|
||||
|
||||
@@ -12,12 +12,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def should_split(node: torch.fx.Node, splitting_ops: list[str]) -> bool:
|
||||
"""
|
||||
Check if a node should be split for dynamo graph partition.
|
||||
"""Check if a node should be split for dynamo graph partition.
|
||||
It operates on dynamo graph, so the node.target can be anything.
|
||||
We need to check and split only on OpOverload and OpOverloadPacket.
|
||||
"""
|
||||
|
||||
if node.op != "call_function":
|
||||
return False
|
||||
|
||||
@@ -50,6 +48,7 @@ def inductor_partition_rule_context(
|
||||
|
||||
Args:
|
||||
splitting_ops: List of operator names to partition on.
|
||||
|
||||
"""
|
||||
if not splitting_ops:
|
||||
logger.debug("No partition ops provided; skipping rule registration.")
|
||||
|
||||
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
|
||||
|
||||
|
||||
class ActivationQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
Base class for Activation+Quant fusions.
|
||||
"""Base class for Activation+Quant fusions.
|
||||
Should not be used directly.
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Fp8StaticQuant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kFp8StaticTensorSym)
|
||||
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Nvfp4Quant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kNvfp4Dynamic)
|
||||
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
Supports group_size 128 and 64 via QuantKey.
|
||||
Parameterized on is_scale_transposed for different scale layouts.
|
||||
"""
|
||||
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
|
||||
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
|
||||
|
||||
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
|
||||
"""
|
||||
for node in match.nodes:
|
||||
if node.target == _IR_RMS_NORM_OP:
|
||||
x, weight = node.args[0], node.args[1]
|
||||
@@ -347,8 +348,7 @@ class BasePattern:
|
||||
|
||||
|
||||
class AllReduceRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm before attn in the first Transformer block.
|
||||
"""
|
||||
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
|
||||
"""
|
||||
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
|
||||
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
|
||||
|
||||
|
||||
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for Attention+Fp8StaticQuant.
|
||||
"""Fusion for Attention+Fp8StaticQuant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class AttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for Attention+Nvfp4Quant.
|
||||
"""Fusion for Attention+Nvfp4Quant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
|
||||
|
||||
|
||||
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto attention if supported.
|
||||
"""This pass fuses post-attention quantization onto attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
|
||||
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
"""Utility for inputs to the pattern"""
|
||||
"""Utility for inputs to the pattern."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8StaticQuant.
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Nvfp4Quant.
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -34,8 +34,7 @@ P = ParamSpec("P")
|
||||
|
||||
|
||||
class QkNormRopePattern:
|
||||
"""
|
||||
Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
"""Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
|
||||
Unfused (conceptually):
|
||||
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
|
||||
|
||||
@@ -106,8 +106,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class QkNormRopeKvCachePattern:
|
||||
"""
|
||||
Match the unfused sequence:
|
||||
"""Match the unfused sequence:
|
||||
q, k, v = split(qkv, ...)
|
||||
q = rms_norm(q.view(heads), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads), k_weight).view(flat)
|
||||
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
|
||||
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
|
||||
|
||||
class FusedRMSQuantKey(NamedTuple):
|
||||
"""
|
||||
Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
"""Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
quant: type of quantization
|
||||
fused_add: does the op also perform the residual add
|
||||
"""
|
||||
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
|
||||
|
||||
|
||||
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
|
||||
|
||||
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
"""This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
ops into an aiter rms_norm_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
into a aiter rms_norm_with_add_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
|
||||
independent fused ``rms_norm_group_fp8_quant`` ops.
|
||||
|
||||
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
|
||||
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
|
||||
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
|
||||
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
with rocm_aiter_fused_rms_gated_fp8_group_quant.
|
||||
|
||||
The norm operates per-head on (N*H, D) tensors. The compiler folds the
|
||||
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
|
||||
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
ops into an aiter silu_and_mul_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
|
||||
|
||||
class AddAiterRMSNormPadPattern:
|
||||
"""
|
||||
This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
with a custom triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
|
||||
|
||||
|
||||
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
with an triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
class MLADualRMSNormPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
AITER's ``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
|
||||
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
],
|
||||
]
|
||||
):
|
||||
"""
|
||||
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
|
||||
|
||||
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
|
||||
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
|
||||
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
is_neox: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This impl fetches the KV cache and slot mapping from the forward context,
|
||||
"""This impl fetches the KV cache and slot mapping from the forward context,
|
||||
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
|
||||
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
|
||||
that is passed to unified_attention to signal a side effect and
|
||||
@@ -98,8 +97,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class RopeStaticQQuantKVCachePattern:
|
||||
"""
|
||||
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
dependency ordering.
|
||||
"""
|
||||
|
||||
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
|
||||
|
||||
|
||||
class RopeReshapeKVCachePattern:
|
||||
"""
|
||||
This pattern matches the following unfused inplace ops:
|
||||
"""This pattern matches the following unfused inplace ops:
|
||||
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
|
||||
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
|
||||
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
|
||||
|
||||
|
||||
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses the rotary embedding and KV cache update operations
|
||||
"""This pass fuses the rotary embedding and KV cache update operations
|
||||
into a single fused kernel if available.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
|
||||
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
|
||||
tp_size: int,
|
||||
element_size: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Calculate the minimum token threshold for applying sequence parallelism.
|
||||
"""Calculate the minimum token threshold for applying sequence parallelism.
|
||||
|
||||
Returns None if sequence parallelism should not be applied based on model size.
|
||||
|
||||
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
|
||||
|
||||
class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass enables sequence parallelism for models.
|
||||
"""This pass enables sequence parallelism for models.
|
||||
It identifies patterns where an AllReduce operation is followed by
|
||||
an RMSNorm (or RMSNorm and then Quantization) operation.
|
||||
These patterns are replaced with a ReduceScatter operation, followed by
|
||||
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
"""
|
||||
Determines if sequence parallelism should be applied for the given
|
||||
"""Determines if sequence parallelism should be applied for the given
|
||||
compile range.
|
||||
|
||||
SP is only beneficial for larger batch sizes where the communication
|
||||
|
||||
@@ -64,14 +64,12 @@ def _hash_source_cached(*srcs: str | type | types.FunctionType) -> str:
|
||||
|
||||
|
||||
class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
A custom graph pass that uses a hash of its source as the UUID.
|
||||
"""A custom graph pass that uses a hash of its source as the UUID.
|
||||
This is defined as a convenience and should work in most cases.
|
||||
"""
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
"""Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
This should depend on the pass implementation, so that changes to the
|
||||
pass result in recompilation.
|
||||
By default, the object source is hashed.
|
||||
@@ -80,14 +78,14 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_source(*srcs: str | Any) -> str:
|
||||
"""
|
||||
Utility method to hash the sources of functions or objects.
|
||||
"""Utility method to hash the sources of functions or objects.
|
||||
|
||||
Args:
|
||||
srcs: strings or objects to add to the hash.
|
||||
Objects and functions have their source inspected.
|
||||
Results are cached by resolved types to avoid repeated
|
||||
inspect.getsource() calls.
|
||||
|
||||
"""
|
||||
# Resolve instances to their class for a hashable cache key.
|
||||
cache_key = tuple(
|
||||
@@ -98,11 +96,11 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_dict(dict_: dict[Any, Any]) -> str:
|
||||
"""
|
||||
Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
"""Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
|
||||
Returns:
|
||||
A sha256 hash of the json rep of the dictionary.
|
||||
|
||||
"""
|
||||
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
@@ -112,8 +110,7 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
|
||||
class CallableInductorPass(InductorPass):
|
||||
"""
|
||||
This class is a wrapper for a callable that automatically provides an
|
||||
"""This class is a wrapper for a callable that automatically provides an
|
||||
implementation of the UUID.
|
||||
"""
|
||||
|
||||
@@ -131,8 +128,7 @@ class CallableInductorPass(InductorPass):
|
||||
|
||||
|
||||
def enable_fake_mode(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
"""Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
create or run things with real tensors.
|
||||
"""
|
||||
|
||||
|
||||
@@ -70,8 +70,7 @@ def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
|
||||
|
||||
|
||||
class UnsafeCloneEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
"""This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
|
||||
contents of non-donated graph inputs.
|
||||
|
||||
|
||||
@@ -19,8 +19,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
"""This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
The maybe_inplace overloads have the same signature as the default overload
|
||||
so the pass simply replaces the called overload.
|
||||
That makes the graph properly functional.
|
||||
|
||||
@@ -23,9 +23,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRLoweringPass(VllmInductorPass):
|
||||
"""
|
||||
This pass lowers vLLM IR ops to their implementations the priority list.
|
||||
"""
|
||||
"""This pass lowers vLLM IR ops to their implementations the priority list."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
super().__init__(vllm_config)
|
||||
@@ -113,8 +111,7 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
IR op priority & impl sources affect lowering pass output,
|
||||
"""IR op priority & impl sources affect lowering pass output,
|
||||
so we include them in the cache key.
|
||||
"""
|
||||
priorities = {name: op.get_priority() for name, op in IrOp.registry.items()}
|
||||
|
||||
@@ -71,8 +71,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Function decorator that turns on inductor pattern match debug
|
||||
"""Function decorator that turns on inductor pattern match debug
|
||||
for the duration of the call.
|
||||
Used to avoid logging builtin Inductor pattern matching.
|
||||
"""
|
||||
@@ -89,8 +88,7 @@ def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
|
||||
|
||||
class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
The pass manager for post-grad passes.
|
||||
"""The pass manager for post-grad passes.
|
||||
It handles configuration, adding custom passes, and running passes.
|
||||
It supports uuid for the Inductor code cache. That includes torch<2.6
|
||||
support using pickling (in .inductor_pass.CustomGraphPass).
|
||||
@@ -236,8 +234,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes.append(pass_)
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
"""The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
affects compilation caching. Its uuid depends on the UUIDs of all
|
||||
dependent passes and the pass config. See InductorPass for more info.
|
||||
"""
|
||||
|
||||
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FixFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
"""This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
After this pass, DCE (dead-code elimination) should never be run,
|
||||
as de-functionalized nodes may appear as dead code.
|
||||
|
||||
@@ -260,9 +259,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self.nodes_to_remove.clear()
|
||||
|
||||
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]) -> None:
|
||||
"""
|
||||
Stage a node (or nodes) for removal at the end of the pass.
|
||||
"""
|
||||
"""Stage a node (or nodes) for removal at the end of the pass."""
|
||||
if isinstance(node_or_nodes, torch.fx.Node):
|
||||
self.nodes_to_remove.append(node_or_nodes)
|
||||
else:
|
||||
@@ -275,8 +272,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
mutated_args: dict[int, torch.fx.Node | str],
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
De-functionalize a node by replacing it with a call to the original.
|
||||
"""De-functionalize a node by replacing it with a call to the original.
|
||||
It also replaces the getitem users with the mutated arguments.
|
||||
See replace_users_with_mutated_args and insert_defunctionalized.
|
||||
"""
|
||||
@@ -287,14 +283,14 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
def replace_users_with_mutated_args(
|
||||
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
|
||||
) -> None:
|
||||
"""
|
||||
Replace mutated getitem users of the auto-functionalized node with the
|
||||
"""Replace mutated getitem users of the auto-functionalized node with the
|
||||
mutated arguments.
|
||||
|
||||
Args:
|
||||
node: The auto-functionalized node
|
||||
mutated_args: The mutated arguments, indexed by getitem index.
|
||||
If the value of an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
"""
|
||||
for idx, user in self.getitem_users(node).items():
|
||||
# Some functionalized nodes may return both a result at getitem[0]
|
||||
@@ -310,8 +306,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self._remove(user)
|
||||
|
||||
def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]:
|
||||
"""
|
||||
Returns the operator.getitem users of the auto-functionalized node,
|
||||
"""Returns the operator.getitem users of the auto-functionalized node,
|
||||
indexed by the index they are getting.
|
||||
"""
|
||||
users = {}
|
||||
@@ -327,8 +322,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: torch.fx.Node,
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Insert a new defunctionalized node into the graph before node.
|
||||
"""Insert a new defunctionalized node into the graph before node.
|
||||
If one of the kwargs is 'out', provide args directly,
|
||||
as node.kwargs cannot be used.
|
||||
See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
|
||||
@@ -338,6 +332,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: The auto-functionalized node to defunctionalize
|
||||
args: If we cannot use kwargs, specify args directly.
|
||||
If an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
""" # noqa: E501
|
||||
assert is_func(node, auto_functionalized), (
|
||||
f"node must be auto-functionalized, is {node} instead"
|
||||
|
||||
@@ -16,8 +16,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NoOpEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This is an inductor pass that removes redundant reshape/slice operations.
|
||||
"""This is an inductor pass that removes redundant reshape/slice operations.
|
||||
It is required for RMSNorm-quant fusion to work properly.
|
||||
That's because apply_fp8_linear adds a reshape, which is redundant
|
||||
in the 2D-case. Additionally, torch internal no-op elimination pass does
|
||||
@@ -106,8 +105,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
|
||||
# ---------------------- Shape comparison helpers ----------------------
|
||||
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
|
||||
"""
|
||||
This function checks if two dimensions are equivalent.
|
||||
"""This function checks if two dimensions are equivalent.
|
||||
|
||||
Args:
|
||||
dim: The dimension arg to reshape/slice
|
||||
@@ -119,6 +117,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
There are two cases in which the dimensions are equivalent:
|
||||
1. The dimensions are equal (both integers)
|
||||
2. The dimensions both correspond to the same SymInt
|
||||
|
||||
"""
|
||||
# Case 1
|
||||
return statically_known_true(dim == i_dim) # type: ignore[no-any-return]
|
||||
|
||||
@@ -6,8 +6,7 @@ from ..vllm_inductor_pass import VllmInductorPass
|
||||
|
||||
|
||||
class PostCleanupPass(VllmInductorPass):
|
||||
"""
|
||||
This pass performs cleanup after custom passes.
|
||||
"""This pass performs cleanup after custom passes.
|
||||
It topologically sorts the graph and removes unused nodes.
|
||||
This is needed because the pattern matcher does not guarantee producing
|
||||
a topologically sorted graph, and there may be unused nodes left around.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
"""Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
assignment if there are no users for the inplace tensor written to by
|
||||
the slice_scatter call.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
input tensor with the same split sizes.
|
||||
|
||||
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
|
||||
@@ -10,9 +9,10 @@ that CSE fails to merge. This pass detects and replaces the duplicates
|
||||
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
|
||||
see a single split node with all users attached.
|
||||
|
||||
See also:
|
||||
See Also:
|
||||
- vLLM #33295 (original issue)
|
||||
- PyTorch #174472 (upstream CSE gap)
|
||||
|
||||
"""
|
||||
|
||||
import operator
|
||||
@@ -30,7 +30,8 @@ logger = init_logger(__name__)
|
||||
|
||||
class SplitCoalescingPass(VllmInductorPass):
|
||||
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
|
||||
node when they share the same input tensor and split sizes."""
|
||||
node when they share the same input tensor and split sizes.
|
||||
"""
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
|
||||
@@ -35,8 +35,7 @@ class InductorCompilationConfig:
|
||||
|
||||
|
||||
class VllmInductorPass(InductorPass):
|
||||
"""
|
||||
An inductor pass with access to vLLM PassConfig.
|
||||
"""An inductor pass with access to vLLM PassConfig.
|
||||
It provides timing, logging, and dumping utilities.
|
||||
"""
|
||||
|
||||
@@ -93,8 +92,7 @@ def get_match_table() -> dict[str, int]:
|
||||
|
||||
|
||||
class VllmPatternMatcherPass(VllmInductorPass):
|
||||
"""
|
||||
A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
"""A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
Provides pattern registration with match counting, debug dumping, and logging.
|
||||
"""
|
||||
|
||||
@@ -109,7 +107,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
)
|
||||
|
||||
def _replace_op_overloads(self, string: str) -> str:
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations"""
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations."""
|
||||
return str(
|
||||
self._OP_OVERLOAD_PATTERN.sub(
|
||||
lambda m: f"torch.ops.{m.group(1)}.{m.group(2)}",
|
||||
@@ -123,8 +121,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
logger.debug("fusion pass matches: %s", dict(cls.match_table))
|
||||
|
||||
def dump_patterns(self, config: VllmConfig, pm_pass: PatternMatcherPass) -> None:
|
||||
"""
|
||||
If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
"""If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
into the debug_dump_path folder next to the dumped fx graphs.
|
||||
|
||||
This method does its best to print something that looks like Python code
|
||||
@@ -195,8 +192,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
"""
|
||||
A pattern/replacement pair for FX graph fusion.
|
||||
"""A pattern/replacement pair for FX graph fusion.
|
||||
|
||||
Implement the three abstract members below, then pass
|
||||
instances to VllmFusionPatternMatcherPass.register(). The pass will
|
||||
@@ -216,8 +212,7 @@ class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
@property
|
||||
@abstractmethod
|
||||
def replacement(self) -> Callable[P, R]:
|
||||
"""
|
||||
Returns a closure defining the FX subgraph to
|
||||
"""Returns a closure defining the FX subgraph to
|
||||
substitute in place of each match.
|
||||
"""
|
||||
...
|
||||
@@ -294,8 +289,7 @@ def _remove_noop_permutes(gm: fx.GraphModule) -> None:
|
||||
|
||||
|
||||
class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
"""A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
Subclasses register patterns via self.register() in their own __init__.
|
||||
"""
|
||||
|
||||
|
||||
@@ -96,8 +96,7 @@ class PiecewiseBackend:
|
||||
compiled_runnables: dict[str, Callable[..., Any]] | None = None,
|
||||
submod_name: str = "",
|
||||
):
|
||||
"""
|
||||
The backend for piecewise compilation.
|
||||
"""The backend for piecewise compilation.
|
||||
It mainly handles the compilation of static shapes and
|
||||
dispatching based on runtime shape.
|
||||
|
||||
|
||||
@@ -45,8 +45,7 @@ def _compilation_context() -> Generator[None, None, None]:
|
||||
|
||||
|
||||
class TorchCompileWithNoGuardsWrapper:
|
||||
"""
|
||||
A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
"""A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
|
||||
When guards are dropped, the first time __call__ is invoked, a single
|
||||
compilation is triggered. Dynamo should never be traced again after that
|
||||
@@ -272,8 +271,7 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
@contextmanager
|
||||
def _dispatch_to_compiled_code(self) -> Generator[None, None, None]:
|
||||
# noqa: E501
|
||||
"""
|
||||
Context manager to dispatch to internally compiled code for torch<2.8.
|
||||
"""Context manager to dispatch to internally compiled code for torch<2.8.
|
||||
Why does this work? Because Dynamo guarantees that the compiled
|
||||
bytecode has exactly the same arguments, cell variables, and free
|
||||
variables as the original code. Therefore we can directly switch
|
||||
@@ -291,9 +289,7 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
|
||||
|
||||
def reset_compile_wrapper(model: torch.nn.Module) -> None:
|
||||
"""
|
||||
Clean up compiled model and captured CUDA graphs for elastic EP.
|
||||
"""
|
||||
"""Clean up compiled model and captured CUDA graphs for elastic EP."""
|
||||
if not isinstance(model, TorchCompileWithNoGuardsWrapper) and hasattr(
|
||||
model, "model"
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user