Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -16,8 +16,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
|
||||
class MambaBase(AttentionLayerBase):
|
||||
"""
|
||||
Base class for Mamba-like layers which support the v1 engine.
|
||||
"""Base class for Mamba-like layers which support the v1 engine.
|
||||
Inherit from this class if you implement a custom layer.
|
||||
"""
|
||||
|
||||
@@ -44,8 +43,7 @@ class MambaBase(AttentionLayerBase):
|
||||
|
||||
@abstractmethod
|
||||
def get_state_shape(self) -> Iterable[tuple[int, ...]]:
|
||||
"""
|
||||
Defines the shape of the state.
|
||||
"""Defines the shape of the state.
|
||||
For mamba layers this is usually a (conv_state, ssm_state) tuple.
|
||||
In this case, returns (conv_state_shape, ssm_state_shape).
|
||||
"""
|
||||
|
||||
@@ -45,8 +45,7 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
|
||||
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"""
|
||||
Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
|
||||
This implements the linear attention mechanism that replaces sliding window
|
||||
attention in the hybrid architecture.
|
||||
@@ -294,9 +293,7 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Core attention computation (called by custom op).
|
||||
"""
|
||||
"""Core attention computation (called by custom op)."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
|
||||
@@ -639,9 +639,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkvz: torch.Tensor,
|
||||
mixed_ba: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Derives `query`, `key` and `value` tensors from `mixed_qkvzba`.
|
||||
"""
|
||||
"""Derives `query`, `key` and `value` tensors from `mixed_qkvzba`."""
|
||||
new_tensor_shape_qkvz = mixed_qkvz.size()[:-1] + (
|
||||
self.num_k_heads // self.tp_size,
|
||||
(
|
||||
@@ -692,8 +690,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_ba: torch.Tensor,
|
||||
num_tokens: int,
|
||||
):
|
||||
"""
|
||||
Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
|
||||
"""Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
|
||||
|
||||
For gqa_interleaved_layout (Qwen3-Next): unpack the interleaved
|
||||
[ng, (hk + hk + np/ng*hv + np/ng*hv)] layout into contiguous qkv.
|
||||
@@ -864,7 +861,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
available, otherwise falling back to the generic CUDA path.
|
||||
"""
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
@@ -899,8 +897,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with three parts:
|
||||
"""Forward pass with three parts:
|
||||
1. Input projection
|
||||
2. Core attention (custom op)
|
||||
3. Output projection
|
||||
@@ -977,8 +974,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with three parts:
|
||||
"""Forward pass with three parts:
|
||||
1. Input projection
|
||||
2. Core attention (custom op)
|
||||
3. Output projection
|
||||
@@ -1221,6 +1217,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
z_out: **output** buffer for z (num_tokens, num_heads,
|
||||
head_dim); mutated in-place.
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1279,6 +1276,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
b: beta gating vector (num_tokens, num_heads)
|
||||
a: alpha gating vector (num_tokens, num_heads)
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1658,9 +1656,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
core_attn_out: torch.Tensor,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
):
|
||||
"""
|
||||
Core attention computation with a packed non-spec decode fast path.
|
||||
"""
|
||||
"""Core attention computation with a packed non-spec decode fast path."""
|
||||
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501
|
||||
self_kv_cache = self.kv_cache
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
@@ -2030,8 +2026,7 @@ def fused_gdn_gating(
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Fused computation of g and beta for Gated Delta Net.
|
||||
"""Fused computation of g and beta for Gated Delta Net.
|
||||
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
beta_output = b.sigmoid()
|
||||
TODO maybe use torch.compile to replace this triton kernel
|
||||
|
||||
@@ -523,7 +523,7 @@ class BailingMoELinearAttention(LinearAttention):
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Forward method called by torch.ops.vllm.linear_attention"""
|
||||
"""Forward method called by torch.ops.vllm.linear_attention."""
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
|
||||
@@ -49,8 +49,7 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
# --8<-- [start:mamba_mixer]
|
||||
@PluggableLayer.register("mamba_mixer")
|
||||
class MambaMixer(MambaBase, PluggableLayer):
|
||||
"""
|
||||
Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
@@ -237,8 +236,7 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
"""
|
||||
Run the Mamba-1 SSM pipeline.
|
||||
"""Run the Mamba-1 SSM pipeline.
|
||||
|
||||
Steps
|
||||
-----
|
||||
@@ -258,7 +256,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
decode tokens), both sets of kernels are executed independently
|
||||
and their outputs are concatenated before the final output projection.
|
||||
"""
|
||||
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
|
||||
@@ -252,8 +252,7 @@ def mamba_v2_sharded_weight_loader(
|
||||
# --8<-- [start:mamba_mixer2]
|
||||
@PluggableLayer.register("mamba_mixer2")
|
||||
class MambaMixer2(MambaBase, PluggableLayer):
|
||||
"""
|
||||
Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
@@ -1221,7 +1220,6 @@ def share_replayssm_ring_trackers(
|
||||
The final local layer in each group advances its cursors after every layer
|
||||
in that group has consumed the previous values.
|
||||
"""
|
||||
|
||||
replayssm_mixers: dict[str, MambaMixer2] = {}
|
||||
for layer_name in ordered_layer_names:
|
||||
layer = forward_context[layer_name]
|
||||
|
||||
@@ -261,8 +261,8 @@ class MambaStateShapeCalculator:
|
||||
@classmethod
|
||||
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
|
||||
"""Compute the increase in group numbers to account for
|
||||
replication in order to accompany the head shards."""
|
||||
|
||||
replication in order to accompany the head shards.
|
||||
"""
|
||||
# in the case ngoups % tp_size == 0, this will be zero
|
||||
if ngroups % tp_size == 0:
|
||||
return 0
|
||||
@@ -344,13 +344,13 @@ class MambaStateShapeCalculator:
|
||||
|
||||
@dataclass
|
||||
class MambaCopySpec:
|
||||
"""
|
||||
Data class specifying the memory-copy parameters for Mamba states used for
|
||||
"""Data class specifying the memory-copy parameters for Mamba states used for
|
||||
prefix caching in align mode.
|
||||
|
||||
Attributes:
|
||||
start_addr (int): Starting address for the memory copy operation.
|
||||
num_elements (int): Number of elements to copy from the starting address.
|
||||
|
||||
"""
|
||||
|
||||
start_addr: int
|
||||
|
||||
@@ -497,7 +497,7 @@ def causal_conv1d_fn(
|
||||
metadata=None,
|
||||
validate_data=False,
|
||||
):
|
||||
"""support varlen + continuous batching when x is 2D tensor
|
||||
"""Support varlen + continuous batching when x is 2D tensor.
|
||||
|
||||
x: (dim,cu_seq_len)
|
||||
cu_seq_len = total tokens of all seqs in that batch
|
||||
@@ -1109,8 +1109,7 @@ def causal_conv1d_update(
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
x: Input tensor which can take the following shapes:
|
||||
"""x: Input tensor which can take the following shapes:
|
||||
|
||||
- `[batch, dim]` - single token prediction
|
||||
- `[batch, dim, seqlen]` - single or multiple tokens prediction
|
||||
|
||||
@@ -123,8 +123,7 @@ def causal_conv1d_update_torch(
|
||||
bias: torch.Tensor | None = None,
|
||||
activation: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pure PyTorch fallback for causal_conv1d_update.
|
||||
"""Pure PyTorch fallback for causal_conv1d_update.
|
||||
Currently used as a fallback for Arm (aarch64) to leverage
|
||||
oneDNN/ACL F.conv1d kernels for batched decoding.
|
||||
"""
|
||||
|
||||
@@ -173,6 +173,7 @@ def chunk_gated_delta_rule_cutedsl(
|
||||
``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``.
|
||||
When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of
|
||||
that buffer.
|
||||
|
||||
"""
|
||||
q = q.squeeze(0)
|
||||
k = k.squeeze(0)
|
||||
|
||||
@@ -728,7 +728,6 @@ def h_cutedsl(
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
"""Compute H/V_new with the same argument order as the CUDA wrapper."""
|
||||
|
||||
_, H, K_dim = K.shape
|
||||
_, Hv, V_dim = V.shape
|
||||
h_dtype = {
|
||||
|
||||
@@ -59,7 +59,8 @@ def get_ssm_device_name() -> str:
|
||||
|
||||
def _canonical_cache_dtype(cache_dtype: str) -> str:
|
||||
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
|
||||
configs because the kernel only sees bit width when accessing state."""
|
||||
configs because the kernel only sees bit width when accessing state.
|
||||
"""
|
||||
return "float16" if cache_dtype == "bfloat16" else cache_dtype
|
||||
|
||||
|
||||
@@ -67,8 +68,7 @@ def _canonical_cache_dtype(cache_dtype: str) -> str:
|
||||
def get_ssm_configs(
|
||||
headdim: int, dstate: int, cache_dtype: str
|
||||
) -> dict[int, Any] | None:
|
||||
"""
|
||||
Return tuned (BLOCK_SIZE_M, num_warps) configs for *selective_state_update*
|
||||
"""Return tuned (BLOCK_SIZE_M, num_warps) configs for *selective_state_update*
|
||||
keyed by ``effective_batch = batch * nheads``, or ``None`` if no config
|
||||
file is found for the (headdim, dstate, cache_dtype, device) combination.
|
||||
|
||||
@@ -161,7 +161,8 @@ _ssm_config_override: tuple[int, int] | None = None
|
||||
@contextmanager
|
||||
def override_ssm_config(config: tuple[int, int]):
|
||||
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
|
||||
the context. Used by the tuning benchmark to time specific configs."""
|
||||
the context. Used by the tuning benchmark to time specific configs.
|
||||
"""
|
||||
global _ssm_config_override
|
||||
prev = _ssm_config_override
|
||||
_ssm_config_override = config
|
||||
@@ -515,30 +516,29 @@ def selective_state_update(
|
||||
enable_stochastic_rounding=False,
|
||||
cache_philox_rounds=0,
|
||||
):
|
||||
"""
|
||||
Argument:
|
||||
state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
|
||||
x: (batch, dim) or (batch, nheads, dim)
|
||||
dt: (batch, dim) or (batch, nheads, dim)
|
||||
A: (dim, dstate) or (nheads, dim, dstate)
|
||||
B: (batch, dstate) or (batch, ngroups, dstate)
|
||||
C: (batch, dstate) or (batch, ngroups, dstate)
|
||||
D: (dim,) or (nheads, dim)
|
||||
z: (batch, dim) or (batch, nheads, dim)
|
||||
dt_bias: (dim,) or (nheads, dim)
|
||||
null_block_id: int
|
||||
if state_batch_indices is passed, lets the kernel identify
|
||||
padded entries that will not be processed,
|
||||
for example: state_batch_indices = [null_block_id, 1, 20,
|
||||
null_block_id] in this case, the kernel will not process
|
||||
entries at indices 0 and 3
|
||||
out: Preallocated ssm output tensor. Assume same shape as x.
|
||||
In-place updated.
|
||||
num_accepted_tokens: (batch,)
|
||||
number of accepted tokens from previous verification step,
|
||||
tells the kernel which initial state to use
|
||||
cu_seqlens: (batch,)
|
||||
length per sequence, for variable length in speculative decoding cases
|
||||
"""Argument:
|
||||
state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
|
||||
x: (batch, dim) or (batch, nheads, dim)
|
||||
dt: (batch, dim) or (batch, nheads, dim)
|
||||
A: (dim, dstate) or (nheads, dim, dstate)
|
||||
B: (batch, dstate) or (batch, ngroups, dstate)
|
||||
C: (batch, dstate) or (batch, ngroups, dstate)
|
||||
D: (dim,) or (nheads, dim)
|
||||
z: (batch, dim) or (batch, nheads, dim)
|
||||
dt_bias: (dim,) or (nheads, dim)
|
||||
null_block_id: int
|
||||
if state_batch_indices is passed, lets the kernel identify
|
||||
padded entries that will not be processed,
|
||||
for example: state_batch_indices = [null_block_id, 1, 20,
|
||||
null_block_id] in this case, the kernel will not process
|
||||
entries at indices 0 and 3
|
||||
out: Preallocated ssm output tensor. Assume same shape as x.
|
||||
In-place updated.
|
||||
num_accepted_tokens: (batch,)
|
||||
number of accepted tokens from previous verification step,
|
||||
tells the kernel which initial state to use
|
||||
cu_seqlens: (batch,)
|
||||
length per sequence, for variable length in speculative decoding cases
|
||||
"""
|
||||
if state.dim() == 3:
|
||||
state = state.unsqueeze(1)
|
||||
@@ -723,8 +723,7 @@ def selective_scan_fn(
|
||||
cu_chunk_seqlen=None,
|
||||
last_chunk_indices=None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
u: (dim, total_length) for varlen or (batch, dim, seqlen)
|
||||
"""u: (dim, total_length) for varlen or (batch, dim, seqlen)
|
||||
applies changes in place.
|
||||
ssm_states: (batch, dim, dstate) or (batch, nheads, dim, dstate)
|
||||
applies changes in place.
|
||||
@@ -769,9 +768,12 @@ def selective_scan_fn(
|
||||
initial_state_idx: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the cache block
|
||||
containing the initial state is located.
|
||||
returns
|
||||
|
||||
Returns
|
||||
-------
|
||||
output: (dim, total_length) for varlen or (batch, dim, seqlen)
|
||||
supports inplace replacement
|
||||
|
||||
"""
|
||||
if u.stride(-1) != 1:
|
||||
u = u.contiguous()
|
||||
|
||||
@@ -146,16 +146,17 @@ def _bmm_chunk_fwd_kernel(
|
||||
|
||||
|
||||
def _bmm_chunk_fwd(a, b, chunk_size, cu_chunk_seqlens, causal=False, output_dtype=None):
|
||||
"""
|
||||
Argument:
|
||||
"""Argument:
|
||||
a: (seqlen, ngroups, k)
|
||||
b: (seqlen, ngroups, k)
|
||||
chunk_size: int
|
||||
cu_chunk_seq_lens: (nchunks+1,)
|
||||
causal: if True, then out[i, j] for i > j will be arbitrary, only out[i, j] for i <= j are
|
||||
guaranteed to be correct.
|
||||
|
||||
Return:
|
||||
out: (nchunks, ngroups, chunk_size, chunk_size)
|
||||
|
||||
"""
|
||||
seqlen, ngroups, k = a.shape
|
||||
assert b.shape == a.shape
|
||||
|
||||
@@ -175,8 +175,7 @@ def mamba_chunk_scan_combined_varlen(
|
||||
return_intermediate_states=False,
|
||||
state_dtype=None,
|
||||
):
|
||||
"""
|
||||
Argument:
|
||||
"""Argument:
|
||||
x: (seqlen, nheads, headdim)
|
||||
dt: (seqlen, nheads)
|
||||
A: (nheads)
|
||||
@@ -198,7 +197,6 @@ def mamba_chunk_scan_combined_varlen(
|
||||
Return:
|
||||
varlen_states: (batch, nheads, headdim, dstate)
|
||||
"""
|
||||
|
||||
assert cu_seqlens is not None, "cu_seqlens must be provided assuming varlen input"
|
||||
assert seq_idx is not None
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Dispatch module for Mamba selective state update (SSU) backends.
|
||||
"""Dispatch module for Mamba selective state update (SSU) backends.
|
||||
|
||||
Provides a unified `selective_state_update` function that dispatches to
|
||||
the Triton, FlashInfer, or CPU backend based on the configured
|
||||
|
||||
Reference in New Issue
Block a user