[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
+1 -2
View File
@@ -1059,8 +1059,7 @@ def _fused_eh_norm_kernel(
):
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
"""
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
+1 -2
View File
@@ -1425,8 +1425,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -91,8 +91,7 @@ def validate_image_sentinel_ids(tokenizer) -> None:
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
"""Number of LLM tokens the aligner grid occupies (N-layout, including
row/align padding).
"""
row/align padding)."""
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
num_tokens = n_llm_h * (n_llm_w + 1) + 2
@@ -197,8 +196,7 @@ def load_image(
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
"""Builds the N-layout token types (final order) and the aligner-row order
for IMAGE slots.
"""
for IMAGE slots."""
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
pad_h = n_llm_h % 2
rows = n_llm_h + pad_h
@@ -241,8 +239,7 @@ def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
class DeepseekV4VLImageProcessor:
"""Per-image transform (the PIL-input equivalent of the reference
``load_image``).
"""
``load_image``)."""
def __init__(self, config: DeepseekV4Config) -> None:
super().__init__()
@@ -312,8 +312,7 @@ class FusedIndexerQRopeQuantTritonKernel(
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
(`SparseMQAIndexer`) is enabled.
"""
(`SparseMQAIndexer`) is enabled."""
if vllm_config.attention_config.indexer_sparse_logits:
return (torch.float32, torch.bfloat16)
return (torch.float32,)
@@ -42,8 +42,7 @@ class DeepseekV4CPUCompressor(DeepseekCompressor):
weight loading (see
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
kernel requires fp32/contiguous but the checkpoint loads this weight
in bf16, and it never changes after loading.
"""
in bf16, and it never changes after loading."""
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
def forward(
+1 -2
View File
@@ -118,8 +118,7 @@ class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
) -> dict[str, FlashMLASchedMeta | None]:
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
C++ decode-path concern) -- always return the all-``None`` sentinel
the shared base's own CPU branch returns.
"""
the shared base's own CPU branch returns."""
return {
_LAYER_TYPE_SWAONLY: None,
_LAYER_TYPE_C4A: None,
+8 -16
View File
@@ -176,15 +176,13 @@ def _fused_indexer_q_rope_quant_cpu(
class _NoOpEvent:
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
stream list, these events are constructed but never actually used.
"""
stream list, these events are constructed but never actually used."""
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
"""CPU indexer K-cache descriptor: same fields as the shared base, just
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``.
"""
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV4CPUIndexerBackend
@@ -211,8 +209,7 @@ class DeepseekV4CPUIndexer(DeepseekV4Indexer):
compressor run straight-line instead of through
``maybe_execute_in_parallel``; the short-context check drops the
CUDA-only ``is_current_stream_capturing()`` guard (always False
here).
"""
here)."""
compressor = self.compressor
attn_metadata = get_forward_context().attn_metadata
@@ -312,8 +309,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
``bmm_cpu`` before the FP8 kernel's own
``process_weights_after_loading`` VNNI-repacks it in place for
row-major reads ``_o_proj`` never does.
"""
row-major reads ``_o_proj`` never does."""
wo_a = self.wo_a
quant_method = wo_a.quant_method
orig_pwal = quant_method.process_weights_after_loading
@@ -404,8 +400,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> torch.Tensor:
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
supported here, so the base method's bf16/per-tensor-fp8 branches
are dropped.
"""
are dropped."""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
@@ -450,8 +445,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""CPU override: two ``RMSNorm`` calls instead of the shared
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
link under triton-cpu (``undefined symbol: __truncdfbf2``).
"""
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
return self.q_norm(qr), None, self.kv_norm(kv)
@@ -469,8 +463,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> None:
"""CPU override: no aux streams, so query-projection+KV-insert, the
indexer, and the compressor run straight-line instead of through
execute_in_parallel/maybe_execute_in_parallel.
"""
execute_in_parallel/maybe_execute_in_parallel."""
attn_metadata = get_forward_context().attn_metadata
indexer = self.indexer
compressor = self.compressor
@@ -519,8 +512,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
out: torch.Tensor,
) -> None:
"""CPU override: identical body, minus ``@eager_break_during_capture``
(this platform never captures a CUDA graph, so it was a no-op).
"""
(this platform never captures a CUDA graph, so it was a no-op)."""
if self.indexer is not None and index_q is not None:
assert index_weights is not None
# index_q_scale is always None on CPU (FP8-only path).
+1 -2
View File
@@ -907,8 +907,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -159,8 +159,7 @@ class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
def _realloc_nvfp4_params(self) -> None:
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
fp8-e4m3 per-16 block scales plus the per-tensor second-level
scales (weight_scale_2, input_scale) modelopt exports.
"""
scales (weight_scale_2, input_scale) modelopt exports."""
n_e = self.num_local_experts
inter = self.intermediate_size
hidden = self.hidden_size
@@ -56,8 +56,7 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
def _packed_block_span(pool: torch.Tensor) -> int:
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
block_size for unpacked KV, larger when packed (#44577). Raises if not
token-aligned.
"""
token-aligned."""
block_stride = pool.stride(0)
token_stride = pool.stride(-2)
if block_stride % token_stride != 0:
+1 -2
View File
@@ -1985,8 +1985,7 @@ class DeepseekV4ForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -1378,8 +1378,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA)
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -4
View File
@@ -1062,8 +1062,7 @@ class DeepseekV41LLMForCausalLM(
@property
def token_lookback_depth(self) -> int:
"""Tokens before a chunk start the engram hash needs; the model runner
passes them as `lookback_token_ids`.
"""
passes them as `lookback_token_ids`."""
engram_hash = self.model.engram_hash
return engram_hash.lookback_depth if engram_hash is not None else 0
@@ -1087,8 +1086,7 @@ class DeepseekV41LLMForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -209,8 +209,7 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
self, image_embeds: torch.Tensor, types: torch.Tensor
) -> torch.Tensor:
"""Full image span: aligner rows at IMAGE slots, the learned
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
"""
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
types = types.to(image_embeds.device)
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
dtype = image_embeds.dtype
+1 -2
View File
@@ -1066,8 +1066,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
def _compressed_kv_cache(self) -> torch.Tensor:
"""The compressed-KV cache tensor of this layer's kv source (own
cache for kv-source layers).
"""
cache for kv-source layers)."""
if self.is_kv_source:
return self.kv_cache
assert self.compressed_cache_prefix is not None
+3 -6
View File
@@ -564,8 +564,7 @@ def _engram_head_shard_weight_loader(
param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
"""Load this rank's complete head buckets. ue8m0 scales arrive as
float8_e8m0fnu; keep the raw bytes (the param stores uint8).
"""
float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
part_rows = param.shape[0]
if loaded_weight.dtype == torch.float8_e8m0fnu:
loaded_weight = loaded_weight.view(torch.uint8)
@@ -740,8 +739,7 @@ class ParallelEngramEmbedding(nn.Module):
def forward(self, indices: torch.Tensor) -> torch.Tensor:
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
bf16, gathered from all shards for this replica's tokens.
"""
bf16, gathered from all shards for this replica's tokens."""
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
@@ -1002,8 +1000,7 @@ class Engram(nn.Module):
) -> torch.Tensor:
"""hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
tokens, pre sequence-parallel shard); token_mask: [T], False shuts
the gate so those positions pass through untouched.
"""
the gate so those positions pass through untouched."""
kv = self.wkv(self.embed(hash_ids).flatten(-2))
num_kv_tokens = hash_ids.shape[0]
assert token_mask is None or token_mask.shape == (num_kv_tokens,)
@@ -82,8 +82,7 @@ def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int:
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
"""Largest aspect-preserving pixel size whose token grid still fits in
max_n_token. Returns (best_height, best_width).
"""
max_n_token. Returns (best_height, best_width)."""
r = height / width
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
max_h_float = max_w_float * r
@@ -106,8 +105,7 @@ def safe_resize(
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
):
"""Shrink the pixel size until the image costs at most max_n_token LLM
tokens.
"""
tokens."""
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
best_height, best_width = solve_resize_ratio(
@@ -173,8 +171,7 @@ def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor:
class DeepseekV4VLImageProcessor:
"""Per-image transform (the PIL-input equivalent of the reference
``load_image``).
"""
``load_image``)."""
def __init__(self, config: DeepseekV41Config) -> None:
super().__init__()
@@ -54,8 +54,7 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
def _packed_block_span(pool: torch.Tensor) -> int:
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
block_size for unpacked KV, larger when packed (#44577). Raises if not
token-aligned.
"""
token-aligned."""
block_stride = pool.stride(0)
token_stride = pool.stride(-2)
if block_stride % token_stride != 0:
+2 -4
View File
@@ -1179,8 +1179,7 @@ class DeepseekV41LLMForCausalLM(
@property
def token_lookback_depth(self) -> int:
"""Tokens before a chunk start the engram hash needs; the model runner
passes them as `lookback_token_ids`.
"""
passes them as `lookback_token_ids`."""
engram_hash = self.model.engram_hash
return engram_hash.lookback_depth if engram_hash is not None else 0
@@ -1204,8 +1203,7 @@ class DeepseekV41LLMForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step.
"""
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -209,8 +209,7 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
self, image_embeds: torch.Tensor, types: torch.Tensor
) -> torch.Tensor:
"""Full image span: aligner rows at IMAGE slots, the learned
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
"""
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
types = types.to(image_embeds.device)
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
dtype = image_embeds.dtype
@@ -98,8 +98,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
BLOCK_D: tl.constexpr,
):
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
"""
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
row = tl.program_id(0)
do_write = True
if HAS_WRITE_MASK:
+2 -4
View File
@@ -130,8 +130,7 @@ def _resolve_kda_prefill_backend(
"""Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on
SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate) or the Triton
``chunk_kda_with_fused_gate`` path. ``backend`` comes from
``additional_config.kda_prefill_backend`` (auto / triton / flashkda).
"""
``additional_config.kda_prefill_backend`` (auto / triton / flashkda)."""
if backend not in ("auto", "triton", "flashkda"):
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
capability = current_platform.get_device_capability()
@@ -369,8 +368,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
bounded gate ``lower_bound * sigmoid(exp(A_log) * (g + dt_bias))``,
matching ``chunk_kda_with_fused_gate(..., safe_gate=True)``. Writes the
attention output into ``out`` (a workspace buffer when ``None``) and
returns ``(out, final_state)``.
"""
returns ``(out, final_state)``."""
assert self._flashkda_buffer_specs is not None
final_state, workspace, workspace_out = (
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
+1 -2
View File
@@ -720,8 +720,7 @@ class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
(image token repeat, video frame/timestamp structure) is owned by vLLM's
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
the replacement content and the placeholder scan validates against
exactly that.
"""
exactly that."""
def _hf_processor_applies_updates(
self,
@@ -138,8 +138,7 @@ def _gather_workspace_shapes(
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
"""
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
@@ -157,8 +156,7 @@ def kv_cache_as_quant_view(
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation.
"""
DeepGEMM, from the 3D indexer kv-cache allocation."""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
@@ -31,8 +31,7 @@ def _fused_eh_norm_kernel(
):
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
"""
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
@@ -163,8 +163,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
BLOCK_D: tl.constexpr,
):
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
"""
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
row = tl.program_id(0)
do_write = True
if HAS_WRITE_MASK:
+1 -2
View File
@@ -72,8 +72,7 @@ def _sconv_add_norm(
ROCm uses the portable collective path. The Lamport P2P implementation in
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
publication semantics which cannot be linked into a gfx950 HSACO.
"""
publication semantics which cannot be linked into a gfx950 HSACO."""
norm_w = norm.weight if norm is not None else None
eps = norm.variance_epsilon if norm is not None else 0.0
+3 -6
View File
@@ -192,8 +192,7 @@ def inkling_gate_select(
class InklingGate(nn.Module):
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
global scale (the served checkpoint's only configuration).
"""
global scale (the served checkpoint's only configuration)."""
def __init__(
self,
@@ -247,8 +246,7 @@ class InklingGate(nn.Module):
self, gating_output: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
are the routed top-k; the S trailing entries are the sink gammas.
"""
are the routed top-k; the S trailing entries are the sink gammas."""
return inkling_gate_select(
gating_output,
self.n_total_experts,
@@ -269,8 +267,7 @@ class InklingGate(nn.Module):
def _inkling_moe_ep_size() -> int:
"""EP size the MoERunner layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled).
"""
expert parallelism is enabled)."""
parallel_config = get_current_vllm_config().parallel_config
if not parallel_config.enable_expert_parallel:
return 1
+3 -6
View File
@@ -166,8 +166,7 @@ class InklingMultiTokenPredictor(nn.Module):
) -> torch.Tensor:
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
the target's tower embeddings scattered in unnormed (the backbone
convention — MM embeds are merged after embed_norm).
"""
convention — MM embeds are merged after embed_norm)."""
norm = self.backbone_embed_norm
embeds = embed_rmsnorm(
input_ids,
@@ -196,8 +195,7 @@ class InklingMultiTokenPredictor(nn.Module):
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
launch: embedding row gather + the backbone embed_norm + the depth
embed_norm chain on one side, hidden_norm on the other, written
straight into the cat buffer.
"""
straight into the cat buffer."""
hidden_w = layer.hidden_norm.weight
embed_w = layer.embed_norm.weight
eps = layer.hidden_norm.variance_epsilon
@@ -329,8 +327,7 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
reduction — no full-vocab logits all-gather. The muP divisor is a
positive scalar, so the argmax is invariant and the scaling is
skipped entirely.
"""
skipped entirely."""
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -4
View File
@@ -238,8 +238,7 @@ def embed_rmsnorm(
embedding-table gather.
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
first decoder layer's pre-attention norm) as a second output, still one
launch. Bit-exact vs the unfused module sequence.
"""
launch. Bit-exact vs the unfused module sequence."""
ids = input_ids.view(-1)
(T,) = ids.shape
n = embed_table.shape[1]
@@ -333,8 +332,7 @@ def embed_dual_rmsnorm_cat(
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
unfused sequence). The concat copies collapse into direct writes.
"""
unfused sequence). The concat copies collapse into direct writes."""
T, n = hidden.shape
if embeds is not None:
assert embeds.shape == hidden.shape
+1 -2
View File
@@ -75,8 +75,7 @@ def _sconv_add_norm(
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
The Lamport path performs reduce-scatter + shard sconv + all-gather +
residual add + norm. The NCCL path handles unsupported configurations.
"""
residual add + norm. The NCCL path handles unsupported configurations."""
attn_metadata = get_forward_context().attn_metadata
m = (
attn_metadata.get(sconv.owner.prefix)
+3 -6
View File
@@ -191,8 +191,7 @@ def inkling_gate_select(
class InklingGate(nn.Module):
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
global scale (the served checkpoint's only configuration).
"""
global scale (the served checkpoint's only configuration)."""
def __init__(
self,
@@ -244,8 +243,7 @@ class InklingGate(nn.Module):
self, gating_output: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
are the routed top-k; the S trailing entries are the sink gammas.
"""
are the routed top-k; the S trailing entries are the sink gammas."""
return inkling_gate_select(
gating_output,
self.n_total_experts,
@@ -266,8 +264,7 @@ class InklingGate(nn.Module):
def _inkling_moe_ep_size() -> int:
"""EP size the FusedMoE layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled).
"""
expert parallelism is enabled)."""
parallel_config = get_current_vllm_config().parallel_config
if not parallel_config.enable_expert_parallel:
return 1
+3 -6
View File
@@ -166,8 +166,7 @@ class InklingMultiTokenPredictor(nn.Module):
) -> torch.Tensor:
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
the target's tower embeddings scattered in unnormed (the backbone
convention — MM embeds are merged after embed_norm).
"""
convention — MM embeds are merged after embed_norm)."""
norm = self.backbone_embed_norm
embeds = embed_rmsnorm(
input_ids,
@@ -196,8 +195,7 @@ class InklingMultiTokenPredictor(nn.Module):
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
launch: embedding row gather + the backbone embed_norm + the depth
embed_norm chain on one side, hidden_norm on the other, written
straight into the cat buffer.
"""
straight into the cat buffer."""
hidden_w = layer.hidden_norm.weight
embed_w = layer.embed_norm.weight
eps = layer.hidden_norm.variance_epsilon
@@ -329,8 +327,7 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
reduction — no full-vocab logits all-gather. The muP divisor is a
positive scalar, so the argmax is invariant and the scaling is
skipped entirely.
"""
skipped entirely."""
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -4
View File
@@ -60,8 +60,7 @@ def dmel_embed_sum_norm(
eps: float,
) -> torch.Tensor:
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
[T, NB, D] intermediate).
"""
[T, NB, D] intermediate)."""
T, nb = dmel_idx.shape
D = weight.shape[1]
assert weight.shape[0] % nb == 0
@@ -152,8 +151,7 @@ def rmsnorm_gelu(
) -> torch.Tensor:
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
comes back as ``fold_timespace_to_depth(result, *fold)``.
"""
comes back as ``fold_timespace_to_depth(result, *fold)``."""
D = x.shape[-1]
flat = x.reshape(-1, D)
assert flat.stride(1) == 1 and flat.stride(0) == D
+2 -4
View File
@@ -238,8 +238,7 @@ def embed_rmsnorm(
embedding-table gather.
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
first decoder layer's pre-attention norm) as a second output, still one
launch. Bit-exact vs the unfused module sequence.
"""
launch. Bit-exact vs the unfused module sequence."""
ids = input_ids.view(-1)
(T,) = ids.shape
n = embed_table.shape[1]
@@ -333,8 +332,7 @@ def embed_dual_rmsnorm_cat(
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
unfused sequence). The concat copies collapse into direct writes.
"""
unfused sequence). The concat copies collapse into direct writes."""
T, n = hidden.shape
if embeds is not None:
assert embeds.shape == hidden.shape
+1 -2
View File
@@ -745,8 +745,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
slot_mapping: torch.Tensor,
) -> torch.Tensor:
"""Fused decode query-concat + latent cache insert, dispatched by cache
dtype (same policy as prefill: fp8 cache -> fp8 query).
"""
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
if self.kv_cache_dtype == "fp8_ds_mla":
cache = self.kv_cache
if cache.dtype != torch.uint8:
+1 -2
View File
@@ -334,8 +334,7 @@ def _aiter_moe_fused_shared_experts_enabled(
class MiniMaxM3MoE(nn.Module):
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
expert.
"""
expert."""
def __init__(
self,
+2 -4
View File
@@ -173,8 +173,7 @@ class MiniMaxM3IndexerPrefillMetadata:
@dataclass
class MiniMaxM3IndexerDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens).
"""
per-request query length (1, or 1 + num_speculative_tokens)."""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
@@ -207,8 +206,7 @@ class MiniMaxM3IndexerMetadataBuilder(
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
):
"""Abstract base: shared setup only. The Triton and MSA builders are
parallel subclasses that each own their full ``build`` (no shared code).
"""
parallel subclasses that each own their full ``build`` (no shared code)."""
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
@@ -203,8 +203,7 @@ class MiniMaxM3SparsePrefillMetadata:
@dataclass
class MiniMaxM3SparseDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens).
"""
per-request query length (1, or 1 + num_speculative_tokens)."""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
+2 -4
View File
@@ -85,8 +85,7 @@ class MiniMaxM3IndexerMSAPrefillMetadata:
class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
"""Decode reuses the inherited base ``decode`` field (the Triton decode
metadata); ``prefill_msa`` carries the fmha score plan for the prefill side
(the base ``prefill`` field is unused on this path).
"""
(the base ``prefill`` field is unused on this path)."""
prefill_msa: MiniMaxM3IndexerMSAPrefillMetadata | None = None
# Per-forward view (``[:num_tokens]``) of the builder's persistent unified
@@ -112,8 +111,7 @@ class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
"""Decode metadata is the cudagraph-safe Triton decode metadata; the prefill
fmha plan is built eagerly (prefill batches are not captured).
"""
fmha plan is built eagerly (prefill batches are not captured)."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
+2 -4
View File
@@ -190,8 +190,7 @@ class MiniMaxM3MLP(nn.Module):
class MiniMaxM3MoE(nn.Module):
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
expert.
"""
expert."""
def __init__(
self,
@@ -783,8 +782,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
@property
def ffn_all_reduce_deferred(self) -> bool:
"""This layer's FFN output is left un-reduced; the caller fuses the
all-reduce into the next RMSNorm.
"""
all-reduce into the next RMSNorm."""
if self.is_moe_layer:
return self.block_sparse_moe.experts.moe_config.skip_final_all_reduce
return not self.mlp.down_proj.reduce_results