[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -1059,8 +1059,7 @@ def _fused_eh_norm_kernel(
|
||||
):
|
||||
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
|
||||
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
|
||||
"""
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -1425,8 +1425,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -91,8 +91,7 @@ def validate_image_sentinel_ids(tokenizer) -> None:
|
||||
|
||||
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
|
||||
"""Number of LLM tokens the aligner grid occupies (N-layout, including
|
||||
row/align padding).
|
||||
"""
|
||||
row/align padding)."""
|
||||
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
|
||||
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
|
||||
num_tokens = n_llm_h * (n_llm_w + 1) + 2
|
||||
@@ -197,8 +196,7 @@ def load_image(
|
||||
|
||||
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
|
||||
"""Builds the N-layout token types (final order) and the aligner-row order
|
||||
for IMAGE slots.
|
||||
"""
|
||||
for IMAGE slots."""
|
||||
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
|
||||
pad_h = n_llm_h % 2
|
||||
rows = n_llm_h + pad_h
|
||||
@@ -241,8 +239,7 @@ def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
|
||||
|
||||
class DeepseekV4VLImageProcessor:
|
||||
"""Per-image transform (the PIL-input equivalent of the reference
|
||||
``load_image``).
|
||||
"""
|
||||
``load_image``)."""
|
||||
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -312,8 +312,7 @@ class FusedIndexerQRopeQuantTritonKernel(
|
||||
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
|
||||
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
|
||||
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
|
||||
(`SparseMQAIndexer`) is enabled.
|
||||
"""
|
||||
(`SparseMQAIndexer`) is enabled."""
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
return (torch.float32, torch.bfloat16)
|
||||
return (torch.float32,)
|
||||
|
||||
@@ -42,8 +42,7 @@ class DeepseekV4CPUCompressor(DeepseekCompressor):
|
||||
weight loading (see
|
||||
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
|
||||
kernel requires fp32/contiguous but the checkpoint loads this weight
|
||||
in bf16, and it never changes after loading.
|
||||
"""
|
||||
in bf16, and it never changes after loading."""
|
||||
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -118,8 +118,7 @@ class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
) -> dict[str, FlashMLASchedMeta | None]:
|
||||
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
|
||||
C++ decode-path concern) -- always return the all-``None`` sentinel
|
||||
the shared base's own CPU branch returns.
|
||||
"""
|
||||
the shared base's own CPU branch returns."""
|
||||
return {
|
||||
_LAYER_TYPE_SWAONLY: None,
|
||||
_LAYER_TYPE_C4A: None,
|
||||
|
||||
@@ -176,15 +176,13 @@ def _fused_indexer_q_rope_quant_cpu(
|
||||
|
||||
class _NoOpEvent:
|
||||
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
|
||||
stream list, these events are constructed but never actually used.
|
||||
"""
|
||||
stream list, these events are constructed but never actually used."""
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
|
||||
"""CPU indexer K-cache descriptor: same fields as the shared base, just
|
||||
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``.
|
||||
"""
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV4CPUIndexerBackend
|
||||
@@ -211,8 +209,7 @@ class DeepseekV4CPUIndexer(DeepseekV4Indexer):
|
||||
compressor run straight-line instead of through
|
||||
``maybe_execute_in_parallel``; the short-context check drops the
|
||||
CUDA-only ``is_current_stream_capturing()`` guard (always False
|
||||
here).
|
||||
"""
|
||||
here)."""
|
||||
compressor = self.compressor
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
@@ -312,8 +309,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
|
||||
``bmm_cpu`` before the FP8 kernel's own
|
||||
``process_weights_after_loading`` VNNI-repacks it in place for
|
||||
row-major reads ``_o_proj`` never does.
|
||||
"""
|
||||
row-major reads ``_o_proj`` never does."""
|
||||
wo_a = self.wo_a
|
||||
quant_method = wo_a.quant_method
|
||||
orig_pwal = quant_method.process_weights_after_loading
|
||||
@@ -404,8 +400,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
|
||||
supported here, so the base method's bf16/per-tensor-fp8 branches
|
||||
are dropped.
|
||||
"""
|
||||
are dropped."""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
@@ -450,8 +445,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""CPU override: two ``RMSNorm`` calls instead of the shared
|
||||
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``).
|
||||
"""
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
return self.q_norm(qr), None, self.kv_norm(kv)
|
||||
|
||||
@@ -469,8 +463,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> None:
|
||||
"""CPU override: no aux streams, so query-projection+KV-insert, the
|
||||
indexer, and the compressor run straight-line instead of through
|
||||
execute_in_parallel/maybe_execute_in_parallel.
|
||||
"""
|
||||
execute_in_parallel/maybe_execute_in_parallel."""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
@@ -519,8 +512,7 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: identical body, minus ``@eager_break_during_capture``
|
||||
(this platform never captures a CUDA graph, so it was a no-op).
|
||||
"""
|
||||
(this platform never captures a CUDA graph, so it was a no-op)."""
|
||||
if self.indexer is not None and index_q is not None:
|
||||
assert index_weights is not None
|
||||
# index_q_scale is always None on CPU (FP8-only path).
|
||||
|
||||
@@ -907,8 +907,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -159,8 +159,7 @@ class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
|
||||
def _realloc_nvfp4_params(self) -> None:
|
||||
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
|
||||
fp8-e4m3 per-16 block scales plus the per-tensor second-level
|
||||
scales (weight_scale_2, input_scale) modelopt exports.
|
||||
"""
|
||||
scales (weight_scale_2, input_scale) modelopt exports."""
|
||||
n_e = self.num_local_experts
|
||||
inter = self.intermediate_size
|
||||
hidden = self.hidden_size
|
||||
|
||||
@@ -56,8 +56,7 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
def _packed_block_span(pool: torch.Tensor) -> int:
|
||||
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
|
||||
block_size for unpacked KV, larger when packed (#44577). Raises if not
|
||||
token-aligned.
|
||||
"""
|
||||
token-aligned."""
|
||||
block_stride = pool.stride(0)
|
||||
token_stride = pool.stride(-2)
|
||||
if block_stride % token_stride != 0:
|
||||
|
||||
@@ -1985,8 +1985,7 @@ class DeepseekV4ForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -1378,8 +1378,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA)
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -1062,8 +1062,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
@property
|
||||
def token_lookback_depth(self) -> int:
|
||||
"""Tokens before a chunk start the engram hash needs; the model runner
|
||||
passes them as `lookback_token_ids`.
|
||||
"""
|
||||
passes them as `lookback_token_ids`."""
|
||||
engram_hash = self.model.engram_hash
|
||||
return engram_hash.lookback_depth if engram_hash is not None else 0
|
||||
|
||||
@@ -1087,8 +1086,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -209,8 +209,7 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
|
||||
self, image_embeds: torch.Tensor, types: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Full image span: aligner rows at IMAGE slots, the learned
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
|
||||
"""
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
|
||||
types = types.to(image_embeds.device)
|
||||
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
|
||||
dtype = image_embeds.dtype
|
||||
|
||||
@@ -1066,8 +1066,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
def _compressed_kv_cache(self) -> torch.Tensor:
|
||||
"""The compressed-KV cache tensor of this layer's kv source (own
|
||||
cache for kv-source layers).
|
||||
"""
|
||||
cache for kv-source layers)."""
|
||||
if self.is_kv_source:
|
||||
return self.kv_cache
|
||||
assert self.compressed_cache_prefix is not None
|
||||
|
||||
@@ -564,8 +564,7 @@ def _engram_head_shard_weight_loader(
|
||||
param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
"""Load this rank's complete head buckets. ue8m0 scales arrive as
|
||||
float8_e8m0fnu; keep the raw bytes (the param stores uint8).
|
||||
"""
|
||||
float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
|
||||
part_rows = param.shape[0]
|
||||
if loaded_weight.dtype == torch.float8_e8m0fnu:
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
@@ -740,8 +739,7 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
|
||||
bf16, gathered from all shards for this replica's tokens.
|
||||
"""
|
||||
bf16, gathered from all shards for this replica's tokens."""
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
@@ -1002,8 +1000,7 @@ class Engram(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
|
||||
tokens, pre sequence-parallel shard); token_mask: [T], False shuts
|
||||
the gate so those positions pass through untouched.
|
||||
"""
|
||||
the gate so those positions pass through untouched."""
|
||||
kv = self.wkv(self.embed(hash_ids).flatten(-2))
|
||||
num_kv_tokens = hash_ids.shape[0]
|
||||
assert token_mask is None or token_mask.shape == (num_kv_tokens,)
|
||||
|
||||
@@ -82,8 +82,7 @@ def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int:
|
||||
|
||||
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
|
||||
"""Largest aspect-preserving pixel size whose token grid still fits in
|
||||
max_n_token. Returns (best_height, best_width).
|
||||
"""
|
||||
max_n_token. Returns (best_height, best_width)."""
|
||||
r = height / width
|
||||
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
|
||||
max_h_float = max_w_float * r
|
||||
@@ -106,8 +105,7 @@ def safe_resize(
|
||||
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
|
||||
):
|
||||
"""Shrink the pixel size until the image costs at most max_n_token LLM
|
||||
tokens.
|
||||
"""
|
||||
tokens."""
|
||||
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
|
||||
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
|
||||
best_height, best_width = solve_resize_ratio(
|
||||
@@ -173,8 +171,7 @@ def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor:
|
||||
|
||||
class DeepseekV4VLImageProcessor:
|
||||
"""Per-image transform (the PIL-input equivalent of the reference
|
||||
``load_image``).
|
||||
"""
|
||||
``load_image``)."""
|
||||
|
||||
def __init__(self, config: DeepseekV41Config) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -54,8 +54,7 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
def _packed_block_span(pool: torch.Tensor) -> int:
|
||||
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
|
||||
block_size for unpacked KV, larger when packed (#44577). Raises if not
|
||||
token-aligned.
|
||||
"""
|
||||
token-aligned."""
|
||||
block_stride = pool.stride(0)
|
||||
token_stride = pool.stride(-2)
|
||||
if block_stride % token_stride != 0:
|
||||
|
||||
@@ -1179,8 +1179,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
@property
|
||||
def token_lookback_depth(self) -> int:
|
||||
"""Tokens before a chunk start the engram hash needs; the model runner
|
||||
passes them as `lookback_token_ids`.
|
||||
"""
|
||||
passes them as `lookback_token_ids`."""
|
||||
engram_hash = self.model.engram_hash
|
||||
return engram_hash.lookback_depth if engram_hash is not None else 0
|
||||
|
||||
@@ -1204,8 +1203,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -209,8 +209,7 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
|
||||
self, image_embeds: torch.Tensor, types: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Full image span: aligner rows at IMAGE slots, the learned
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
|
||||
"""
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
|
||||
types = types.to(image_embeds.device)
|
||||
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
|
||||
dtype = image_embeds.dtype
|
||||
|
||||
@@ -98,8 +98,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
|
||||
"""
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
|
||||
@@ -130,8 +130,7 @@ def _resolve_kda_prefill_backend(
|
||||
"""Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on
|
||||
SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate) or the Triton
|
||||
``chunk_kda_with_fused_gate`` path. ``backend`` comes from
|
||||
``additional_config.kda_prefill_backend`` (auto / triton / flashkda).
|
||||
"""
|
||||
``additional_config.kda_prefill_backend`` (auto / triton / flashkda)."""
|
||||
if backend not in ("auto", "triton", "flashkda"):
|
||||
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
|
||||
capability = current_platform.get_device_capability()
|
||||
@@ -369,8 +368,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
bounded gate ``lower_bound * sigmoid(exp(A_log) * (g + dt_bias))``,
|
||||
matching ``chunk_kda_with_fused_gate(..., safe_gate=True)``. Writes the
|
||||
attention output into ``out`` (a workspace buffer when ``None``) and
|
||||
returns ``(out, final_state)``.
|
||||
"""
|
||||
returns ``(out, final_state)``."""
|
||||
assert self._flashkda_buffer_specs is not None
|
||||
final_state, workspace, workspace_out = (
|
||||
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
|
||||
|
||||
@@ -720,8 +720,7 @@ class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
|
||||
(image token repeat, video frame/timestamp structure) is owned by vLLM's
|
||||
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
|
||||
the replacement content and the placeholder scan validates against
|
||||
exactly that.
|
||||
"""
|
||||
exactly that."""
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
|
||||
@@ -138,8 +138,7 @@ def _gather_workspace_shapes(
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
|
||||
"""
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
@@ -157,8 +156,7 @@ def kv_cache_as_quant_view(
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation.
|
||||
"""
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
|
||||
@@ -31,8 +31,7 @@ def _fused_eh_norm_kernel(
|
||||
):
|
||||
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
|
||||
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
|
||||
"""
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -163,8 +163,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
|
||||
"""
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
|
||||
@@ -72,8 +72,7 @@ def _sconv_add_norm(
|
||||
|
||||
ROCm uses the portable collective path. The Lamport P2P implementation in
|
||||
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
|
||||
publication semantics which cannot be linked into a gfx950 HSACO.
|
||||
"""
|
||||
publication semantics which cannot be linked into a gfx950 HSACO."""
|
||||
norm_w = norm.weight if norm is not None else None
|
||||
eps = norm.variance_epsilon if norm is not None else 0.0
|
||||
|
||||
|
||||
@@ -192,8 +192,7 @@ def inkling_gate_select(
|
||||
|
||||
class InklingGate(nn.Module):
|
||||
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
|
||||
global scale (the served checkpoint's only configuration).
|
||||
"""
|
||||
global scale (the served checkpoint's only configuration)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -247,8 +246,7 @@ class InklingGate(nn.Module):
|
||||
self, gating_output: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
|
||||
are the routed top-k; the S trailing entries are the sink gammas.
|
||||
"""
|
||||
are the routed top-k; the S trailing entries are the sink gammas."""
|
||||
return inkling_gate_select(
|
||||
gating_output,
|
||||
self.n_total_experts,
|
||||
@@ -269,8 +267,7 @@ class InklingGate(nn.Module):
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the MoERunner layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled).
|
||||
"""
|
||||
expert parallelism is enabled)."""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
if not parallel_config.enable_expert_parallel:
|
||||
return 1
|
||||
|
||||
@@ -166,8 +166,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
|
||||
the target's tower embeddings scattered in unnormed (the backbone
|
||||
convention — MM embeds are merged after embed_norm).
|
||||
"""
|
||||
convention — MM embeds are merged after embed_norm)."""
|
||||
norm = self.backbone_embed_norm
|
||||
embeds = embed_rmsnorm(
|
||||
input_ids,
|
||||
@@ -196,8 +195,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
|
||||
launch: embedding row gather + the backbone embed_norm + the depth
|
||||
embed_norm chain on one side, hidden_norm on the other, written
|
||||
straight into the cat buffer.
|
||||
"""
|
||||
straight into the cat buffer."""
|
||||
hidden_w = layer.hidden_norm.weight
|
||||
embed_w = layer.embed_norm.weight
|
||||
eps = layer.hidden_norm.variance_epsilon
|
||||
@@ -329,8 +327,7 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
skipped entirely.
|
||||
"""
|
||||
skipped entirely."""
|
||||
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -238,8 +238,7 @@ def embed_rmsnorm(
|
||||
embedding-table gather.
|
||||
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
|
||||
first decoder layer's pre-attention norm) as a second output, still one
|
||||
launch. Bit-exact vs the unfused module sequence.
|
||||
"""
|
||||
launch. Bit-exact vs the unfused module sequence."""
|
||||
ids = input_ids.view(-1)
|
||||
(T,) = ids.shape
|
||||
n = embed_table.shape[1]
|
||||
@@ -333,8 +332,7 @@ def embed_dual_rmsnorm_cat(
|
||||
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
|
||||
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
|
||||
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
|
||||
unfused sequence). The concat copies collapse into direct writes.
|
||||
"""
|
||||
unfused sequence). The concat copies collapse into direct writes."""
|
||||
T, n = hidden.shape
|
||||
if embeds is not None:
|
||||
assert embeds.shape == hidden.shape
|
||||
|
||||
@@ -75,8 +75,7 @@ def _sconv_add_norm(
|
||||
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
|
||||
|
||||
The Lamport path performs reduce-scatter + shard sconv + all-gather +
|
||||
residual add + norm. The NCCL path handles unsupported configurations.
|
||||
"""
|
||||
residual add + norm. The NCCL path handles unsupported configurations."""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
m = (
|
||||
attn_metadata.get(sconv.owner.prefix)
|
||||
|
||||
@@ -191,8 +191,7 @@ def inkling_gate_select(
|
||||
|
||||
class InklingGate(nn.Module):
|
||||
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
|
||||
global scale (the served checkpoint's only configuration).
|
||||
"""
|
||||
global scale (the served checkpoint's only configuration)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -244,8 +243,7 @@ class InklingGate(nn.Module):
|
||||
self, gating_output: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
|
||||
are the routed top-k; the S trailing entries are the sink gammas.
|
||||
"""
|
||||
are the routed top-k; the S trailing entries are the sink gammas."""
|
||||
return inkling_gate_select(
|
||||
gating_output,
|
||||
self.n_total_experts,
|
||||
@@ -266,8 +264,7 @@ class InklingGate(nn.Module):
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the FusedMoE layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled).
|
||||
"""
|
||||
expert parallelism is enabled)."""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
if not parallel_config.enable_expert_parallel:
|
||||
return 1
|
||||
|
||||
@@ -166,8 +166,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
|
||||
the target's tower embeddings scattered in unnormed (the backbone
|
||||
convention — MM embeds are merged after embed_norm).
|
||||
"""
|
||||
convention — MM embeds are merged after embed_norm)."""
|
||||
norm = self.backbone_embed_norm
|
||||
embeds = embed_rmsnorm(
|
||||
input_ids,
|
||||
@@ -196,8 +195,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
|
||||
launch: embedding row gather + the backbone embed_norm + the depth
|
||||
embed_norm chain on one side, hidden_norm on the other, written
|
||||
straight into the cat buffer.
|
||||
"""
|
||||
straight into the cat buffer."""
|
||||
hidden_w = layer.hidden_norm.weight
|
||||
embed_w = layer.embed_norm.weight
|
||||
eps = layer.hidden_norm.variance_epsilon
|
||||
@@ -329,8 +327,7 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
skipped entirely.
|
||||
"""
|
||||
skipped entirely."""
|
||||
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -60,8 +60,7 @@ def dmel_embed_sum_norm(
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
|
||||
[T, NB, D] intermediate).
|
||||
"""
|
||||
[T, NB, D] intermediate)."""
|
||||
T, nb = dmel_idx.shape
|
||||
D = weight.shape[1]
|
||||
assert weight.shape[0] % nb == 0
|
||||
@@ -152,8 +151,7 @@ def rmsnorm_gelu(
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
|
||||
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
|
||||
comes back as ``fold_timespace_to_depth(result, *fold)``.
|
||||
"""
|
||||
comes back as ``fold_timespace_to_depth(result, *fold)``."""
|
||||
D = x.shape[-1]
|
||||
flat = x.reshape(-1, D)
|
||||
assert flat.stride(1) == 1 and flat.stride(0) == D
|
||||
|
||||
@@ -238,8 +238,7 @@ def embed_rmsnorm(
|
||||
embedding-table gather.
|
||||
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
|
||||
first decoder layer's pre-attention norm) as a second output, still one
|
||||
launch. Bit-exact vs the unfused module sequence.
|
||||
"""
|
||||
launch. Bit-exact vs the unfused module sequence."""
|
||||
ids = input_ids.view(-1)
|
||||
(T,) = ids.shape
|
||||
n = embed_table.shape[1]
|
||||
@@ -333,8 +332,7 @@ def embed_dual_rmsnorm_cat(
|
||||
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
|
||||
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
|
||||
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
|
||||
unfused sequence). The concat copies collapse into direct writes.
|
||||
"""
|
||||
unfused sequence). The concat copies collapse into direct writes."""
|
||||
T, n = hidden.shape
|
||||
if embeds is not None:
|
||||
assert embeds.shape == hidden.shape
|
||||
|
||||
@@ -745,8 +745,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused decode query-concat + latent cache insert, dispatched by cache
|
||||
dtype (same policy as prefill: fp8 cache -> fp8 query).
|
||||
"""
|
||||
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
cache = self.kv_cache
|
||||
if cache.dtype != torch.uint8:
|
||||
|
||||
@@ -334,8 +334,7 @@ def _aiter_moe_fused_shared_experts_enabled(
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
|
||||
expert.
|
||||
"""
|
||||
expert."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -173,8 +173,7 @@ class MiniMaxM3IndexerPrefillMetadata:
|
||||
@dataclass
|
||||
class MiniMaxM3IndexerDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens).
|
||||
"""
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
@@ -207,8 +206,7 @@ class MiniMaxM3IndexerMetadataBuilder(
|
||||
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
|
||||
):
|
||||
"""Abstract base: shared setup only. The Triton and MSA builders are
|
||||
parallel subclasses that each own their full ``build`` (no shared code).
|
||||
"""
|
||||
parallel subclasses that each own their full ``build`` (no shared code)."""
|
||||
|
||||
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
@@ -203,8 +203,7 @@ class MiniMaxM3SparsePrefillMetadata:
|
||||
@dataclass
|
||||
class MiniMaxM3SparseDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens).
|
||||
"""
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
|
||||
@@ -85,8 +85,7 @@ class MiniMaxM3IndexerMSAPrefillMetadata:
|
||||
class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
|
||||
"""Decode reuses the inherited base ``decode`` field (the Triton decode
|
||||
metadata); ``prefill_msa`` carries the fmha score plan for the prefill side
|
||||
(the base ``prefill`` field is unused on this path).
|
||||
"""
|
||||
(the base ``prefill`` field is unused on this path)."""
|
||||
|
||||
prefill_msa: MiniMaxM3IndexerMSAPrefillMetadata | None = None
|
||||
# Per-forward view (``[:num_tokens]``) of the builder's persistent unified
|
||||
@@ -112,8 +111,7 @@ class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
|
||||
|
||||
class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
"""Decode metadata is the cudagraph-safe Triton decode metadata; the prefill
|
||||
fmha plan is built eagerly (prefill batches are not captured).
|
||||
"""
|
||||
fmha plan is built eagerly (prefill batches are not captured)."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
|
||||
@@ -190,8 +190,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
|
||||
expert.
|
||||
"""
|
||||
expert."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -783,8 +782,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
@property
|
||||
def ffn_all_reduce_deferred(self) -> bool:
|
||||
"""This layer's FFN output is left un-reduced; the caller fuses the
|
||||
all-reduce into the next RMSNorm.
|
||||
"""
|
||||
all-reduce into the next RMSNorm."""
|
||||
if self.is_moe_layer:
|
||||
return self.block_sparse_moe.experts.moe_config.skip_final_all_reduce
|
||||
return not self.mlp.down_proj.reduce_results
|
||||
|
||||
Reference in New Issue
Block a user