Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -45,8 +45,7 @@ if current_platform.is_cuda_alike():
|
||||
|
||||
|
||||
class ActivationQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
Base class for Activation+Quant fusions.
|
||||
"""Base class for Activation+Quant fusions.
|
||||
Should not be used directly.
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class ActivationQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Fp8StaticQuant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Fp8StaticQuant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kFp8StaticTensorSym)
|
||||
@@ -126,9 +123,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+Nvfp4Quant Pattern
|
||||
"""
|
||||
"""Fusion for SiluMul+Nvfp4Quant Pattern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(kNvfp4Dynamic)
|
||||
@@ -182,8 +177,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
"""Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
Supports group_size 128 and 64 via QuantKey.
|
||||
Parameterized on is_scale_transposed for different scale layouts.
|
||||
"""
|
||||
@@ -281,8 +275,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
|
||||
|
||||
class ActivationQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
|
||||
@@ -66,7 +66,8 @@ def _view_flashinfer_nvfp4_scale_out_as_int32(
|
||||
|
||||
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.
|
||||
"""
|
||||
for node in match.nodes:
|
||||
if node.target == _IR_RMS_NORM_OP:
|
||||
x, weight = node.args[0], node.args[1]
|
||||
@@ -347,8 +348,7 @@ class BasePattern:
|
||||
|
||||
|
||||
class AllReduceRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm before attn in the first Transformer block.
|
||||
"""
|
||||
@@ -409,8 +409,7 @@ class AllReduceRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn and mlp + rmsnorm before attn.
|
||||
"""
|
||||
@@ -609,8 +608,7 @@ class AllReduceFusedAddGemmaRMSNormPattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -684,8 +682,7 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static fp8 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
@@ -767,8 +764,7 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (without residual)
|
||||
"""This pattern replaces the allreduce + rms norm (without residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to allreduce + rmsnorm + quant before attn
|
||||
in the first Transformer block.
|
||||
@@ -864,8 +860,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
|
||||
|
||||
class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
"""
|
||||
This pattern replaces the allreduce + rms norm (with residual)
|
||||
"""This pattern replaces the allreduce + rms norm (with residual)
|
||||
+ static nvfp4 quant with fused flashinfer implementation.
|
||||
Applies to o_proj + rmsnorm after attn + quant and
|
||||
mlp + rmsnorm + quant before attn.
|
||||
|
||||
@@ -36,8 +36,7 @@ _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=T
|
||||
|
||||
|
||||
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for Attention+Fp8StaticQuant.
|
||||
"""Fusion for Attention+Fp8StaticQuant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -172,8 +171,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class AttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for Attention+Nvfp4Quant.
|
||||
"""Fusion for Attention+Nvfp4Quant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
`fused_output_quant_supported()`. If the pattern is found, the
|
||||
@@ -360,8 +358,7 @@ class AttnNvfp4QuantPattern(
|
||||
|
||||
|
||||
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto attention if supported.
|
||||
"""This pass fuses post-attention quantization onto attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -79,7 +79,7 @@ class MatcherCustomOp(ABC):
|
||||
return torch.empty(*args, dtype=torch.float32, device=self.device, **kwargs)
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
"""Utility for inputs to the pattern"""
|
||||
"""Utility for inputs to the pattern."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -33,8 +33,7 @@ MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8StaticQuant.
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
@@ -174,8 +173,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Nvfp4Quant.
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
@@ -362,8 +360,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
@@ -572,8 +569,7 @@ class MLAAttnFp8GroupQuantPattern(
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
|
||||
@@ -34,8 +34,7 @@ P = ParamSpec("P")
|
||||
|
||||
|
||||
class QkNormRopePattern:
|
||||
"""
|
||||
Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
"""Match the unfused sequence in attention blocks and replace with the fused op.
|
||||
|
||||
Unfused (conceptually):
|
||||
q, k, v = split(qkv, [qsz, kvsz, kvsz], -1)
|
||||
|
||||
@@ -106,8 +106,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class QkNormRopeKvCachePattern:
|
||||
"""
|
||||
Match the unfused sequence:
|
||||
"""Match the unfused sequence:
|
||||
q, k, v = split(qkv, ...)
|
||||
q = rms_norm(q.view(heads), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads), k_weight).view(flat)
|
||||
@@ -413,8 +412,7 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
|
||||
@@ -168,8 +168,7 @@ if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
|
||||
|
||||
class FusedRMSQuantKey(NamedTuple):
|
||||
"""
|
||||
Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
"""Named tuple for identifying the type of RMSNorm + quant fusion.
|
||||
quant: type of quantization
|
||||
fused_add: does the op also perform the residual add
|
||||
"""
|
||||
@@ -771,8 +770,7 @@ class FusedAddRMSNormNvfp4QuantPattern:
|
||||
|
||||
|
||||
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
"""This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
|
||||
|
||||
@@ -180,8 +180,7 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
"""This pattern fuses aiter rms_norm & group fp8 quant custom
|
||||
ops into an aiter rms_norm_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -236,8 +235,7 @@ class AiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
"""This pattern fuses aiter rms_norm_with_add & group fp8 quant custom ops
|
||||
into a aiter rms_norm_with_add_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -298,8 +296,7 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
"""Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
|
||||
independent fused ``rms_norm_group_fp8_quant`` ops.
|
||||
|
||||
@@ -367,8 +364,7 @@ class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
"""View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
|
||||
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
|
||||
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
|
||||
@@ -453,8 +449,7 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
with rocm_aiter_fused_rms_gated_fp8_group_quant.
|
||||
|
||||
The norm operates per-head on (N*H, D) tensors. The compiler folds the
|
||||
@@ -544,8 +539,7 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
into a fused rms_norm_quant op.
|
||||
It also supports fused_add_rms_norm.
|
||||
"""
|
||||
@@ -681,8 +675,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
|
||||
class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
"""
|
||||
This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
"""This pattern fuses aiter silu_and_mul & group fp8 quant custom
|
||||
ops into an aiter silu_and_mul_group_fp8_quant op.
|
||||
"""
|
||||
|
||||
@@ -722,8 +715,7 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
|
||||
|
||||
class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
"""This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
It uses the torch pattern matcher to find the patterns and replace them.
|
||||
|
||||
Because patterns can only be registered once, the pass is a singleton.
|
||||
@@ -744,8 +736,7 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
|
||||
|
||||
class AddAiterRMSNormPadPattern:
|
||||
"""
|
||||
This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
"""This pattern replaces an aiter_rmsnorm_with_add & a pad op
|
||||
with a custom triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -820,8 +811,7 @@ class AddAiterRMSNormPadPattern:
|
||||
|
||||
|
||||
class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
"""This pass replaces an AITER CK RMSNorm + residual add and a pad op
|
||||
with an triton_add_rmsnorm_pad op from AITER.
|
||||
"""
|
||||
|
||||
@@ -854,8 +844,7 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
class MLADualRMSNormPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
"""Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
AITER's ``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
|
||||
@@ -946,8 +935,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
],
|
||||
]
|
||||
):
|
||||
"""
|
||||
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
"""Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
|
||||
|
||||
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
|
||||
@@ -1064,8 +1052,7 @@ class MLADualRMSPerTokenQuantPattern(
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
|
||||
@@ -52,8 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
is_neox: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This impl fetches the KV cache and slot mapping from the forward context,
|
||||
"""This impl fetches the KV cache and slot mapping from the forward context,
|
||||
then calls the layer impl's `AttentionImpl.do_rope_and_kv_cache_update` method.
|
||||
It also returns a dummy tensor, similar to `Attention.unified_kv_cache_update`,
|
||||
that is passed to unified_attention to signal a side effect and
|
||||
@@ -98,8 +97,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class RopeStaticQQuantKVCachePattern:
|
||||
"""
|
||||
Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
"""Fuse rope + static Q fp8 quant while preserving explicit KV-cache update
|
||||
dependency ordering.
|
||||
"""
|
||||
|
||||
@@ -277,8 +275,7 @@ class RopeStaticQQuantKVCachePattern:
|
||||
|
||||
|
||||
class RopeReshapeKVCachePattern:
|
||||
"""
|
||||
This pattern matches the following unfused inplace ops:
|
||||
"""This pattern matches the following unfused inplace ops:
|
||||
q, k = rotary_embedding(positions, q, k, head_size, cos_sin_cache, is_neox)
|
||||
kv_cache_dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
|
||||
@@ -410,8 +407,7 @@ class RopeReshapeKVCachePattern:
|
||||
|
||||
|
||||
class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses the rotary embedding and KV cache update operations
|
||||
"""This pass fuses the rotary embedding and KV cache update operations
|
||||
into a single fused kernel if available.
|
||||
|
||||
It uses the pattern matcher and matches each layer manually, as strings
|
||||
|
||||
@@ -57,8 +57,7 @@ def get_sequence_parallelism_threshold(
|
||||
tp_size: int,
|
||||
element_size: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Calculate the minimum token threshold for applying sequence parallelism.
|
||||
"""Calculate the minimum token threshold for applying sequence parallelism.
|
||||
|
||||
Returns None if sequence parallelism should not be applied based on model size.
|
||||
|
||||
@@ -496,8 +495,7 @@ class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
|
||||
|
||||
class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass enables sequence parallelism for models.
|
||||
"""This pass enables sequence parallelism for models.
|
||||
It identifies patterns where an AllReduce operation is followed by
|
||||
an RMSNorm (or RMSNorm and then Quantization) operation.
|
||||
These patterns are replaced with a ReduceScatter operation, followed by
|
||||
@@ -590,8 +588,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
"""
|
||||
Determines if sequence parallelism should be applied for the given
|
||||
"""Determines if sequence parallelism should be applied for the given
|
||||
compile range.
|
||||
|
||||
SP is only beneficial for larger batch sizes where the communication
|
||||
|
||||
Reference in New Issue
Block a user