Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
|
||||
|
||||
|
||||
class ActivationQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
Base class for Activation+Quant fusions.
|
||||
"""Base class for Activation+Quant fusions.
|
||||
Should not be used directly.
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Fp8StaticQuant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kFp8StaticTensorSym)
|
||||
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Nvfp4Quant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kNvfp4Dynamic)
|
||||
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
Supports group_size 128 and 64 via QuantKey.
|
||||
Parameterized on is_scale_transposed for different scale layouts.
|
||||
"""
|
||||
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
|
||||
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
|
||||
|
||||
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
|
||||
"""
|
||||
for node in match.nodes:
|
||||
if node.target == _IR_RMS_NORM_OP:
|
||||
x, weight = node.args[0], node.args[1]
|
||||
@@ -347,8 +348,7 @@ class BasePattern:
|
||||
|
||||
|
||||
class AllReduceRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm before attn in the first Transformer block.
|
||||
"""
|
||||
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
|
||||
"""
|
||||
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
|
||||
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
|
||||
|
||||
|
||||
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for Attention+Fp8StaticQuant.
|
||||
"""Fusion for Attention+Fp8StaticQuant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class AttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for Attention+Nvfp4Quant.
|
||||
"""Fusion for Attention+Nvfp4Quant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
|
||||
|
||||
|
||||
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto attention if supported.
|
||||
"""This pass fuses post-attention quantization onto attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
|
||||
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
"""Utility for inputs to the pattern"""
|
||||
"""Utility for inputs to the pattern."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8StaticQuant.
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Nvfp4Quant.
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -34,8 +34,7 @@ P = ParamSpec("P")
|
||||
|
||||
|
||||
class QkNormRopePattern:
|
||||
"""
|
||||
Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
"""Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
|
||||
Unfused (conceptually):
|
||||
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
|
||||
|
||||
@@ -106,8 +106,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class QkNormRopeKvCachePattern:
|
||||
"""
|
||||
Match the unfused sequence:
|
||||
"""Match the unfused sequence:
|
||||
q, k, v = split(qkv, ...)
|
||||
q = rms_norm(q.view(heads), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads), k_weight).view(flat)
|
||||
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
|
||||
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
|
||||
|
||||
class FusedRMSQuantKey(NamedTuple):
|
||||
"""
|
||||
Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
"""Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
quant: type of quantization
|
||||
fused_add: does the op also perform the residual add
|
||||
"""
|
||||
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
|
||||
|
||||
|
||||
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
|
||||
|
||||
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
"""This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
ops into an aiter rms_norm_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
into a aiter rms_norm_with_add_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
|
||||
independent fused ``rms_norm_group_fp8_quant`` ops.
|
||||
|
||||
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
|
||||
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
|
||||
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
|
||||
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
with rocm_aiter_fused_rms_gated_fp8_group_quant.
|
||||
|
||||
The norm operates per-head on (N*H, D) tensors. The compiler folds the
|
||||
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
|
||||
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
ops into an aiter silu_and_mul_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
|
||||
|
||||
class AddAiterRMSNormPadPattern:
|
||||
"""
|
||||
This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
with a custom triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
|
||||
|
||||
|
||||
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
with an triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
class MLADualRMSNormPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
AITER's ``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
|
||||
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
],
|
||||
]
|
||||
):
|
||||
"""
|
||||
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
|
||||
|
||||
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
|
||||
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
|
||||
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
is_neox: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This impl fetches the KV cache and slot mapping from the forward context,
|
||||
"""This impl fetches the KV cache and slot mapping from the forward context,
|
||||
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
|
||||
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
|
||||
that is passed to unified_attention to signal a side effect and
|
||||
@@ -98,8 +97,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class RopeStaticQQuantKVCachePattern:
|
||||
"""
|
||||
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
dependency ordering.
|
||||
"""
|
||||
|
||||
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
|
||||
|
||||
|
||||
class RopeReshapeKVCachePattern:
|
||||
"""
|
||||
This pattern matches the following unfused inplace ops:
|
||||
"""This pattern matches the following unfused inplace ops:
|
||||
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
|
||||
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
|
||||
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
|
||||
|
||||
|
||||
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses the rotary embedding and KV cache update operations
|
||||
"""This pass fuses the rotary embedding and KV cache update operations
|
||||
into a single fused kernel if available.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
|
||||
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
|
||||
tp_size: int,
|
||||
element_size: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Calculate the minimum token threshold for applying sequence parallelism.
|
||||
"""Calculate the minimum token threshold for applying sequence parallelism.
|
||||
|
||||
Returns None if sequence parallelism should not be applied based on model size.
|
||||
|
||||
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
|
||||
|
||||
class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass enables sequence parallelism for models.
|
||||
"""This pass enables sequence parallelism for models.
|
||||
It identifies patterns where an AllReduce operation is followed by
|
||||
an RMSNorm (or RMSNorm and then Quantization) operation.
|
||||
These patterns are replaced with a ReduceScatter operation, followed by
|
||||
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
"""
|
||||
Determines if sequence parallelism should be applied for the given
|
||||
"""Determines if sequence parallelism should be applied for the given
|
||||
compile range.
|
||||
|
||||
SP is only beneficial for larger batch sizes where the communication
|
||||
|
||||
@@ -64,14 +64,12 @@ def _hash_source_cached(*srcs: str | type | types.FunctionType) -> str:
|
||||
|
||||
|
||||
class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
A custom graph pass that uses a hash of its source as the UUID.
|
||||
"""A custom graph pass that uses a hash of its source as the UUID.
|
||||
This is defined as a convenience and should work in most cases.
|
||||
"""
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
"""Provide a unique identifier for the pass, used in Inductor code cache.
|
||||
This should depend on the pass implementation, so that changes to the
|
||||
pass result in recompilation.
|
||||
By default, the object source is hashed.
|
||||
@@ -80,14 +78,14 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_source(*srcs: str | Any) -> str:
|
||||
"""
|
||||
Utility method to hash the sources of functions or objects.
|
||||
"""Utility method to hash the sources of functions or objects.
|
||||
|
||||
Args:
|
||||
srcs: strings or objects to add to the hash.
|
||||
Objects and functions have their source inspected.
|
||||
Results are cached by resolved types to avoid repeated
|
||||
inspect.getsource() calls.
|
||||
|
||||
"""
|
||||
# Resolve instances to their class for a hashable cache key.
|
||||
cache_key = tuple(
|
||||
@@ -98,11 +96,11 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
@staticmethod
|
||||
def hash_dict(dict_: dict[Any, Any]) -> str:
|
||||
"""
|
||||
Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
"""Utility method to hash a dictionary, can alternatively be used for uuid.
|
||||
|
||||
Returns:
|
||||
A sha256 hash of the json rep of the dictionary.
|
||||
|
||||
"""
|
||||
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
@@ -112,8 +110,7 @@ class InductorPass(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
|
||||
class CallableInductorPass(InductorPass):
|
||||
"""
|
||||
This class is a wrapper for a callable that automatically provides an
|
||||
"""This class is a wrapper for a callable that automatically provides an
|
||||
implementation of the UUID.
|
||||
"""
|
||||
|
||||
@@ -131,8 +128,7 @@ class CallableInductorPass(InductorPass):
|
||||
|
||||
|
||||
def enable_fake_mode(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
"""Applies a FakeTensorMode context. This is useful when you don't want to
|
||||
create or run things with real tensors.
|
||||
"""
|
||||
|
||||
|
||||
@@ -70,8 +70,7 @@ def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
|
||||
|
||||
|
||||
class UnsafeCloneEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
"""This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
|
||||
contents of non-donated graph inputs.
|
||||
|
||||
|
||||
@@ -19,8 +19,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
"""This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
The maybe_inplace overloads have the same signature as the default overload
|
||||
so the pass simply replaces the called overload.
|
||||
That makes the graph properly functional.
|
||||
|
||||
@@ -23,9 +23,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRLoweringPass(VllmInductorPass):
|
||||
"""
|
||||
This pass lowers vLLM IR ops to their implementations the priority list.
|
||||
"""
|
||||
"""This pass lowers vLLM IR ops to their implementations the priority list."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
super().__init__(vllm_config)
|
||||
@@ -113,8 +111,7 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
IR op priority & impl sources affect lowering pass output,
|
||||
"""IR op priority & impl sources affect lowering pass output,
|
||||
so we include them in the cache key.
|
||||
"""
|
||||
priorities = {name: op.get_priority() for name, op in IrOp.registry.items()}
|
||||
|
||||
@@ -71,8 +71,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
Function decorator that turns on inductor pattern match debug
|
||||
"""Function decorator that turns on inductor pattern match debug
|
||||
for the duration of the call.
|
||||
Used to avoid logging builtin Inductor pattern matching.
|
||||
"""
|
||||
@@ -89,8 +88,7 @@ def with_pattern_match_debug(fn: Callable[P, R]) -> Callable[P, R]:
|
||||
|
||||
|
||||
class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
"""
|
||||
The pass manager for post-grad passes.
|
||||
"""The pass manager for post-grad passes.
|
||||
It handles configuration, adding custom passes, and running passes.
|
||||
It supports uuid for the Inductor code cache. That includes torch<2.6
|
||||
support using pickling (in .inductor_pass.CustomGraphPass).
|
||||
@@ -236,8 +234,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes.append(pass_)
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
"""The PostGradPassManager is set as a custom pass in the Inductor and
|
||||
affects compilation caching. Its uuid depends on the UUIDs of all
|
||||
dependent passes and the pass config. See InductorPass for more info.
|
||||
"""
|
||||
|
||||
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FixFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
"""This pass defunctionalizes certain nodes to avoid redundant tensor copies.
|
||||
After this pass, DCE (dead-code elimination) should never be run,
|
||||
as de-functionalized nodes may appear as dead code.
|
||||
|
||||
@@ -260,9 +259,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self.nodes_to_remove.clear()
|
||||
|
||||
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]) -> None:
|
||||
"""
|
||||
Stage a node (or nodes) for removal at the end of the pass.
|
||||
"""
|
||||
"""Stage a node (or nodes) for removal at the end of the pass."""
|
||||
if isinstance(node_or_nodes, torch.fx.Node):
|
||||
self.nodes_to_remove.append(node_or_nodes)
|
||||
else:
|
||||
@@ -275,8 +272,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
mutated_args: dict[int, torch.fx.Node | str],
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
De-functionalize a node by replacing it with a call to the original.
|
||||
"""De-functionalize a node by replacing it with a call to the original.
|
||||
It also replaces the getitem users with the mutated arguments.
|
||||
See replace_users_with_mutated_args and insert_defunctionalized.
|
||||
"""
|
||||
@@ -287,14 +283,14 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
def replace_users_with_mutated_args(
|
||||
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
|
||||
) -> None:
|
||||
"""
|
||||
Replace mutated getitem users of the auto-functionalized node with the
|
||||
"""Replace mutated getitem users of the auto-functionalized node with the
|
||||
mutated arguments.
|
||||
|
||||
Args:
|
||||
node: The auto-functionalized node
|
||||
mutated_args: The mutated arguments, indexed by getitem index.
|
||||
If the value of an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
"""
|
||||
for idx, user in self.getitem_users(node).items():
|
||||
# Some functionalized nodes may return both a result at getitem[0]
|
||||
@@ -310,8 +306,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
self._remove(user)
|
||||
|
||||
def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]:
|
||||
"""
|
||||
Returns the operator.getitem users of the auto-functionalized node,
|
||||
"""Returns the operator.getitem users of the auto-functionalized node,
|
||||
indexed by the index they are getting.
|
||||
"""
|
||||
users = {}
|
||||
@@ -327,8 +322,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: torch.fx.Node,
|
||||
args: tuple[torch.fx.Node | str, ...] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Insert a new defunctionalized node into the graph before node.
|
||||
"""Insert a new defunctionalized node into the graph before node.
|
||||
If one of the kwargs is 'out', provide args directly,
|
||||
as node.kwargs cannot be used.
|
||||
See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
|
||||
@@ -338,6 +332,7 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
node: The auto-functionalized node to defunctionalize
|
||||
args: If we cannot use kwargs, specify args directly.
|
||||
If an arg is a string, `node.kwargs[arg]` is used.
|
||||
|
||||
""" # noqa: E501
|
||||
assert is_func(node, auto_functionalized), (
|
||||
f"node must be auto-functionalized, is {node} instead"
|
||||
|
||||
@@ -16,8 +16,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NoOpEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This is an inductor pass that removes redundant reshape/slice operations.
|
||||
"""This is an inductor pass that removes redundant reshape/slice operations.
|
||||
It is required for RMSNorm-quant fusion to work properly.
|
||||
That's because apply_fp8_linear adds a reshape, which is redundant
|
||||
in the 2D-case. Additionally, torch internal no-op elimination pass does
|
||||
@@ -106,8 +105,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
|
||||
# ---------------------- Shape comparison helpers ----------------------
|
||||
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
|
||||
"""
|
||||
This function checks if two dimensions are equivalent.
|
||||
"""This function checks if two dimensions are equivalent.
|
||||
|
||||
Args:
|
||||
dim: The dimension arg to reshape/slice
|
||||
@@ -119,6 +117,7 @@ class NoOpEliminationPass(VllmInductorPass):
|
||||
There are two cases in which the dimensions are equivalent:
|
||||
1. The dimensions are equal (both integers)
|
||||
2. The dimensions both correspond to the same SymInt
|
||||
|
||||
"""
|
||||
# Case 1
|
||||
return statically_known_true(dim == i_dim) # type: ignore[no-any-return]
|
||||
|
||||
@@ -6,8 +6,7 @@ from ..vllm_inductor_pass import VllmInductorPass
|
||||
|
||||
|
||||
class PostCleanupPass(VllmInductorPass):
|
||||
"""
|
||||
This pass performs cleanup after custom passes.
|
||||
"""This pass performs cleanup after custom passes.
|
||||
It topologically sorts the graph and removes unused nodes.
|
||||
This is needed because the pattern matcher does not guarantee producing
|
||||
a topologically sorted graph, and there may be unused nodes left around.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
"""Replace ``slice_scatter`` and ``split_with_sizes`` nodes with a single
|
||||
assignment if there are no users for the inplace tensor written to by
|
||||
the slice_scatter call.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
input tensor with the same split sizes.
|
||||
|
||||
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
|
||||
@@ -10,9 +9,10 @@ that CSE fails to merge. This pass detects and replaces the duplicates
|
||||
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
|
||||
see a single split node with all users attached.
|
||||
|
||||
See also:
|
||||
See Also:
|
||||
- vLLM #33295 (original issue)
|
||||
- PyTorch #174472 (upstream CSE gap)
|
||||
|
||||
"""
|
||||
|
||||
import operator
|
||||
@@ -30,7 +30,8 @@ logger = init_logger(__name__)
|
||||
|
||||
class SplitCoalescingPass(VllmInductorPass):
|
||||
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
|
||||
node when they share the same input tensor and split sizes."""
|
||||
node when they share the same input tensor and split sizes.
|
||||
"""
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
|
||||
@@ -35,8 +35,7 @@ class InductorCompilationConfig:
|
||||
|
||||
|
||||
class VllmInductorPass(InductorPass):
|
||||
"""
|
||||
An inductor pass with access to vLLM PassConfig.
|
||||
"""An inductor pass with access to vLLM PassConfig.
|
||||
It provides timing, logging, and dumping utilities.
|
||||
"""
|
||||
|
||||
@@ -93,8 +92,7 @@ def get_match_table() -> dict[str, int]:
|
||||
|
||||
|
||||
class VllmPatternMatcherPass(VllmInductorPass):
|
||||
"""
|
||||
A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
"""A VllmInductorPass that uses the Inductor pattern matcher.
|
||||
Provides pattern registration with match counting, debug dumping, and logging.
|
||||
"""
|
||||
|
||||
@@ -109,7 +107,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
)
|
||||
|
||||
def _replace_op_overloads(self, string: str) -> str:
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations"""
|
||||
"""Replace <OpOverload(..., ...)> with nicer formulations."""
|
||||
return str(
|
||||
self._OP_OVERLOAD_PATTERN.sub(
|
||||
lambda m: f"torch.ops.{m.group(1)}.{m.group(2)}",
|
||||
@@ -123,8 +121,7 @@ class VllmPatternMatcherPass(VllmInductorPass):
|
||||
logger.debug("fusion pass matches: %s", dict(cls.match_table))
|
||||
|
||||
def dump_patterns(self, config: VllmConfig, pm_pass: PatternMatcherPass) -> None:
|
||||
"""
|
||||
If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
"""If debug dumping is enabled, dump the Inductor pattern-matcher patterns
|
||||
into the debug_dump_path folder next to the dumped fx graphs.
|
||||
|
||||
This method does its best to print something that looks like Python code
|
||||
@@ -195,8 +192,7 @@ R = TypeVar("R")
|
||||
|
||||
|
||||
class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
"""
|
||||
A pattern/replacement pair for FX graph fusion.
|
||||
"""A pattern/replacement pair for FX graph fusion.
|
||||
|
||||
Implement the three abstract members below, then pass
|
||||
instances to VllmFusionPatternMatcherPass.register(). The pass will
|
||||
@@ -216,8 +212,7 @@ class VllmPatternReplacement(ABC, Generic[P, R]):
|
||||
@property
|
||||
@abstractmethod
|
||||
def replacement(self) -> Callable[P, R]:
|
||||
"""
|
||||
Returns a closure defining the FX subgraph to
|
||||
"""Returns a closure defining the FX subgraph to
|
||||
substitute in place of each match.
|
||||
"""
|
||||
...
|
||||
@@ -294,8 +289,7 @@ def _remove_noop_permutes(gm: fx.GraphModule) -> None:
|
||||
|
||||
|
||||
class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
"""A VllmPatternMatcherPass for passes that use VllmPatternReplacement objects.
|
||||
Subclasses register patterns via self.register() in their own __init__.
|
||||
"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user