Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+2 -1
View File
@@ -1059,7 +1059,8 @@ def _fused_eh_norm_kernel(
):
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
"""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
+2 -1
View File
@@ -1425,7 +1425,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -496,8 +496,7 @@ class DeepSeekV4MTP(nn.Module):
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DeepseekV4 MLA Attention Layer
"""
"""DeepseekV4 MLA Attention Layer."""
from abc import ABC, abstractmethod
from collections.abc import Callable
@@ -91,7 +91,8 @@ def validate_image_sentinel_ids(tokenizer) -> None:
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
"""Number of LLM tokens the aligner grid occupies (N-layout, including
row/align padding)."""
row/align padding).
"""
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
num_tokens = n_llm_h * (n_llm_w + 1) + 2
@@ -196,7 +197,8 @@ def load_image(
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
"""Builds the N-layout token types (final order) and the aligner-row order
for IMAGE slots."""
for IMAGE slots.
"""
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
pad_h = n_llm_h % 2
rows = n_llm_h + pad_h
@@ -239,7 +241,8 @@ def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
class DeepseekV4VLImageProcessor:
"""Per-image transform (the PIL-input equivalent of the reference
``load_image``)."""
``load_image``).
"""
def __init__(self, config: DeepseekV4Config) -> None:
super().__init__()
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
"""Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
preparation.
- quantize_and_insert_k_cache: quantize bf16 K to UE8M0 FP8 and insert into
@@ -61,8 +60,7 @@ def quantize_and_insert_k_kernel(
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
use_fnuz: tl.constexpr = False,
):
"""
Quantize K tensor and insert into paged K cache.
"""Quantize K tensor and insert into paged K cache.
K Cache block layout (block_size=64 tokens):
- [0, 64*576): Token data, each token has 448 fp8 + 128 bf16
@@ -174,8 +172,7 @@ def quantize_and_insert_k_cache(
is_ue8m0: bool = True,
use_fnuz: bool = False,
):
"""
Quantize K tensor and insert into paged K cache.
"""Quantize K tensor and insert into paged K cache.
K Cache block layout (block_size=64 tokens):
- First 64 * 576 = 36864 bytes: Token data
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Fused compressor + FP8/MXFP4 UE8M0 quantization + KV cache insert kernels.
"""Fused compressor + FP8/MXFP4 UE8M0 quantization + KV cache insert kernels.
Three specialized kernels:
- _fused_kv_compress_norm_rope_insert_sparse_attn:
@@ -390,7 +389,7 @@ def _compress_gather_split_sparse_attn(
NUM_SPLITS: tl.constexpr,
HEAD_TILE: tl.constexpr, # HEAD_SIZE // NUM_SPLITS
):
"""Stage 1: per-(token, head-split) compress gather, write to fp32 scratch
"""Stage 1: per-(token, head-split) compress gather, write to fp32 scratch.
No-overlap gather (cr>=128) on rows [0, COMPRESS_RATIO)
"""
@@ -639,7 +638,7 @@ def compress_norm_rope_store_two_stage_triton(
num_decode_tokens: int,
compress_scratch: torch.Tensor,
) -> None:
"""Two-stage split compressor dispatch for head=512 cr>=128 (no-overlap)
"""Two-stage split compressor dispatch for head=512 cr>=128 (no-overlap).
Run the occupancy-fanned two-stage split for prefill [num_decodee_tokens:]
to fill the CUs, and use the original single-pass launcher
@@ -312,7 +312,8 @@ class FusedIndexerQRopeQuantTritonKernel(
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
(`SparseMQAIndexer`) is enabled."""
(`SparseMQAIndexer`) is enabled.
"""
if vllm_config.attention_config.indexer_sparse_logits:
return (torch.float32, torch.bfloat16)
return (torch.float32,)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
"""Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
@@ -380,6 +379,7 @@ def fused_inv_rope_fp8_quant(
Returns:
Rotated output in [T, G, D] and its FP8 scales. The scale tensor is
empty when quantization is disabled.
"""
from vllm.utils.deep_gemm import get_tma_aligned_size
+1
View File
@@ -263,6 +263,7 @@ def run_dp_sharded_vision_tower(
Returns:
One ``(n_aligner_rows, hidden_size)`` embedding tensor per image.
"""
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
@@ -42,7 +42,8 @@ class DeepseekV4CPUCompressor(DeepseekCompressor):
weight loading (see
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
kernel requires fp32/contiguous but the checkpoint loads this weight
in bf16, and it never changes after loading."""
in bf16, and it never changes after loading.
"""
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
def forward(
+2 -1
View File
@@ -118,7 +118,8 @@ class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
) -> dict[str, FlashMLASchedMeta | None]:
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
C++ decode-path concern) -- always return the all-``None`` sentinel
the shared base's own CPU branch returns."""
the shared base's own CPU branch returns.
"""
return {
_LAYER_TYPE_SWAONLY: None,
_LAYER_TYPE_C4A: None,
+16 -8
View File
@@ -176,13 +176,15 @@ def _fused_indexer_q_rope_quant_cpu(
class _NoOpEvent:
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
stream list, these events are constructed but never actually used."""
stream list, these events are constructed but never actually used.
"""
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
"""CPU indexer K-cache descriptor: same fields as the shared base, just
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``.
"""
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV4CPUIndexerBackend
@@ -209,7 +211,8 @@ class DeepseekV4CPUIndexer(DeepseekV4Indexer):
compressor run straight-line instead of through
``maybe_execute_in_parallel``; the short-context check drops the
CUDA-only ``is_current_stream_capturing()`` guard (always False
here)."""
here).
"""
compressor = self.compressor
attn_metadata = get_forward_context().attn_metadata
@@ -309,7 +312,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
``bmm_cpu`` before the FP8 kernel's own
``process_weights_after_loading`` VNNI-repacks it in place for
row-major reads ``_o_proj`` never does."""
row-major reads ``_o_proj`` never does.
"""
wo_a = self.wo_a
quant_method = wo_a.quant_method
orig_pwal = quant_method.process_weights_after_loading
@@ -400,7 +404,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> torch.Tensor:
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
supported here, so the base method's bf16/per-tensor-fp8 branches
are dropped."""
are dropped.
"""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
@@ -445,7 +450,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""CPU override: two ``RMSNorm`` calls instead of the shared
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
link under triton-cpu (``undefined symbol: __truncdfbf2``).
"""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
return self.q_norm(qr), None, self.kv_norm(kv)
@@ -463,7 +469,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
) -> None:
"""CPU override: no aux streams, so query-projection+KV-insert, the
indexer, and the compressor run straight-line instead of through
execute_in_parallel/maybe_execute_in_parallel."""
execute_in_parallel/maybe_execute_in_parallel.
"""
attn_metadata = get_forward_context().attn_metadata
indexer = self.indexer
compressor = self.compressor
@@ -512,7 +519,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
out: torch.Tensor,
) -> None:
"""CPU override: identical body, minus ``@eager_break_during_capture``
(this platform never captures a CUDA graph, so it was a no-op)."""
(this platform never captures a CUDA graph, so it was a no-op).
"""
if self.indexer is not None and index_q is not None:
assert index_weights is not None
# index_q_scale is always None on CPU (FP8-only path).
+1
View File
@@ -23,6 +23,7 @@ def map_local_to_global_slots_cpu(
Returns:
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
"""
valid = local_indices >= 0
safe_local = local_indices.clamp(min=0).to(torch.int64)
+2 -1
View File
@@ -907,7 +907,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -1
View File
@@ -159,7 +159,8 @@ class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
def _realloc_nvfp4_params(self) -> None:
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
fp8-e4m3 per-16 block scales plus the per-tensor second-level
scales (weight_scale_2, input_scale) modelopt exports."""
scales (weight_scale_2, input_scale) modelopt exports.
"""
n_e = self.num_local_experts
inter = self.intermediate_size
hidden = self.hidden_size
@@ -56,7 +56,8 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
def _packed_block_span(pool: torch.Tensor) -> int:
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
block_size for unpacked KV, larger when packed (#44577). Raises if not
token-aligned."""
token-aligned.
"""
block_stride = pool.stride(0)
token_stride = pool.stride(-2)
if block_stride % token_stride != 0:
+2 -1
View File
@@ -1985,7 +1985,8 @@ class DeepseekV4ForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -518,8 +518,7 @@ class DeepSeekV4MTP(nn.Module):
self.finalize_mega_moe_weights()
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+2 -1
View File
@@ -1378,7 +1378,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA)
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+1 -2
View File
@@ -493,8 +493,7 @@ class DeepSeekV4MTP(nn.Module):
layer.mtp_block.ffn.finalize_mega_moe_weights()
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
+4 -2
View File
@@ -1062,7 +1062,8 @@ class DeepseekV41LLMForCausalLM(
@property
def token_lookback_depth(self) -> int:
"""Tokens before a chunk start the engram hash needs; the model runner
passes them as `lookback_token_ids`."""
passes them as `lookback_token_ids`.
"""
engram_hash = self.model.engram_hash
return engram_hash.lookback_depth if engram_hash is not None else 0
@@ -1086,7 +1087,8 @@ class DeepseekV41LLMForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -1
View File
@@ -209,7 +209,8 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
self, image_embeds: torch.Tensor, types: torch.Tensor
) -> torch.Tensor:
"""Full image span: aligner rows at IMAGE slots, the learned
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
"""
types = types.to(image_embeds.device)
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
dtype = image_embeds.dtype
+3 -4
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DeepseekV4 MLA Attention Layer
"""
"""DeepseekV4 MLA Attention Layer"""
import math
from abc import ABC, abstractmethod
@@ -1068,7 +1066,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
def _compressed_kv_cache(self) -> torch.Tensor:
"""The compressed-KV cache tensor of this layer's kv source (own
cache for kv-source layers)."""
cache for kv-source layers).
"""
if self.is_kv_source:
return self.kv_cache
assert self.compressed_cache_prefix is not None
+6 -3
View File
@@ -564,7 +564,8 @@ def _engram_head_shard_weight_loader(
param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
"""Load this rank's complete head buckets. ue8m0 scales arrive as
float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
float8_e8m0fnu; keep the raw bytes (the param stores uint8).
"""
part_rows = param.shape[0]
if loaded_weight.dtype == torch.float8_e8m0fnu:
loaded_weight = loaded_weight.view(torch.uint8)
@@ -739,7 +740,8 @@ class ParallelEngramEmbedding(nn.Module):
def forward(self, indices: torch.Tensor) -> torch.Tensor:
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
bf16, gathered from all shards for this replica's tokens."""
bf16, gathered from all shards for this replica's tokens.
"""
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
@@ -1000,7 +1002,8 @@ class Engram(nn.Module):
) -> torch.Tensor:
"""hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
tokens, pre sequence-parallel shard); token_mask: [T], False shuts
the gate so those positions pass through untouched."""
the gate so those positions pass through untouched.
"""
kv = self.wkv(self.embed(hash_ids).flatten(-2))
num_kv_tokens = hash_ids.shape[0]
assert token_mask is None or token_mask.shape == (num_kv_tokens,)
@@ -82,7 +82,8 @@ def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int:
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
"""Largest aspect-preserving pixel size whose token grid still fits in
max_n_token. Returns (best_height, best_width)."""
max_n_token. Returns (best_height, best_width).
"""
r = height / width
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
max_h_float = max_w_float * r
@@ -105,7 +106,8 @@ def safe_resize(
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
):
"""Shrink the pixel size until the image costs at most max_n_token LLM
tokens."""
tokens.
"""
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
best_height, best_width = solve_resize_ratio(
@@ -171,7 +173,8 @@ def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor:
class DeepseekV4VLImageProcessor:
"""Per-image transform (the PIL-input equivalent of the reference
``load_image``)."""
``load_image``).
"""
def __init__(self, config: DeepseekV41Config) -> None:
super().__init__()
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
"""Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
preparation.
- quantize_and_insert_k_cache: quantize bf16 K to UE8M0 FP8 and insert into
@@ -76,8 +75,7 @@ def quantize_and_insert_k_kernel(
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
use_fnuz: tl.constexpr = False,
):
"""
Quantize K tensor and insert into paged K cache.
"""Quantize K tensor and insert into paged K cache.
K Cache block layout (block_size=64 tokens):
- [0, 64*576): Token data, each token has 448 fp8 + 128 bf16
@@ -251,8 +249,7 @@ def quantize_and_insert_k_cache(
use_fnuz: bool = False,
bytes_per_token: int = V4_BYTES_PER_TOKEN,
):
"""
Quantize K tensor and insert into paged K cache.
"""Quantize K tensor and insert into paged K cache.
``bytes_per_token`` picks the record (see the module header): the V4 one,
or V4.1's all-dims MXFP8 one.
@@ -58,6 +58,7 @@ def fused_save_compress_norm(
rms_norm_eps: RMSNorm epsilon.
compress_ratio: Group size, either 1 or 2.
latent_out: BF16 [tokens, 512], written only at valid group boundaries.
"""
assert compress_ratio in (1, 2)
assert kv_score.dtype == torch.float32
@@ -46,11 +46,13 @@ def indexer_k_norm_rope_store(
cos_sin_cache: [max_pos, rope_head_dim] GPT-J layout (cos half, then
sin half), from the layer's compress-RoPE instance.
rms_norm_weight: [128] k_norm weight.
rms_norm_eps: Epsilon of the k_norm RMSNorm.
k_cache: uint8 paged indexer cache [num_blocks, block_size, row_bytes].
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
compress_ratio: group size; keys are emitted at group boundaries.
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
per-token FP8 with a single fp32 scale.
"""
num_tokens = kv_slot_mapping.numel()
assert k_pre.ndim == 2 and k_pre.shape[1] == 128
@@ -54,7 +54,8 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
def _packed_block_span(pool: torch.Tensor) -> int:
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
block_size for unpacked KV, larger when packed (#44577). Raises if not
token-aligned."""
token-aligned.
"""
block_stride = pool.stride(0)
token_stride = pool.stride(-2)
if block_stride % token_stride != 0:
+4 -2
View File
@@ -1179,7 +1179,8 @@ class DeepseekV41LLMForCausalLM(
@property
def token_lookback_depth(self) -> int:
"""Tokens before a chunk start the engram hash needs; the model runner
passes them as `lookback_token_ids`."""
passes them as `lookback_token_ids`.
"""
engram_hash = self.model.engram_hash
return engram_hash.lookback_depth if engram_hash is not None else 0
@@ -1203,7 +1204,8 @@ class DeepseekV41LLMForCausalLM(
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
forward(); valid after each target step.
"""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+2 -1
View File
@@ -209,7 +209,8 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
self, image_embeds: torch.Tensor, types: torch.Tensor
) -> torch.Tensor:
"""Full image span: aligner rows at IMAGE slots, the learned
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
"""
types = types.to(image_embeds.device)
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
dtype = image_embeds.dtype
@@ -194,7 +194,7 @@ class WhisperPositionalEmbedding(nn.Embedding):
class WhisperAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
"""Multi-headed attention from 'Attention Is All You Need' paper."""
def __init__(
self,
@@ -346,8 +346,7 @@ class WhisperEncoderLayer(nn.Module):
rotary_cos: torch.Tensor | None = None,
rotary_sin: torch.Tensor | None = None,
) -> tuple[Any, ...]:
"""
Args:
"""Args:
hidden_states: Input to the layer of shape `(batch, seq_len,
embed_dim)`, or `(total_tokens, embed_dim)` when `cu_seqlens_q`
is given.
@@ -365,6 +364,7 @@ class WhisperEncoderLayer(nn.Module):
Returns:
A tuple of the output hidden states, followed by the attention
weights if `output_attentions` is `True`.
"""
residual = hidden_states
hidden_states = self.self_attn_layer_norm(hidden_states)
@@ -435,12 +435,12 @@ class DotsSpeechPreTrainedModel(PreTrainedModel):
class DotsSpeechEncoder(DotsSpeechPreTrainedModel):
"""
Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
"""Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
[`WhisperEncoderLayer`].
Args:
config: WhisperConfig
"""
def __init__(self, config: WhisperConfig):
+13 -1
View File
@@ -98,7 +98,8 @@ def _kpool_softmax_rotate_write_cache_kernel(
BLOCK_D: tl.constexpr,
):
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
"""
row = tl.program_id(0)
do_write = True
if HAS_WRITE_MASK:
@@ -212,6 +213,13 @@ def kpool_compress_and_write_cache(
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
loc: ``[n_pools]`` int64 — flat physical slot per pool.
pool_size: Number of tokens compressed into one cache entry.
head_dim: Indexer head dimension.
write_mask: ``[n_pools]`` bool — pools to write, or None for all.
round_scale: Round each fp8 scale down to a power of two.
return_compressed: Also return the compressed K and scales.
write_cache: Write the compressed result into ``kv_cache``.
"""
assert slot_k.ndim == 3
assert slot_score.shape == slot_k.shape
@@ -592,6 +600,10 @@ def kpool_decode_update_and_maybe_write_cache_batched(
ape: ``[pool_size, head_dim]`` fp32.
slot_mapping: ``[num_requests, next_n]`` int32.
positions: ``[num_requests, next_n]`` int32.
pool_size: Number of tokens compressed into one cache entry.
head_dim: Indexer head dimension.
round_scale: Round each fp8 scale down to a power of two.
"""
num_requests, next_n = key.shape[0], key.shape[1]
if num_requests == 0 or next_n == 0:
@@ -564,8 +564,7 @@ def fused_recurrent_gated_delta_rule(
num_accepted_tokens: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
r"""
Args:
r"""Args:
q (torch.Tensor):
queries of shape `[B, T, H, K]`.
k (torch.Tensor):
@@ -627,6 +626,7 @@ def fused_recurrent_gated_delta_rule(
initial_state=h0,
cu_seqlens=cu_seqlens
)
"""
if cu_seqlens is not None and q.shape[0] != 1:
raise ValueError(
+17 -12
View File
@@ -406,19 +406,24 @@ def chunk_kda_scaled_dot_kkt_fwd(
chunk_size: int = FLA_CHUNK_SIZE,
output_dtype: torch.dtype = torch.float32,
) -> tuple[torch.Tensor, torch.Tensor]:
r"""
Compute beta * K * K^T.
r"""Compute beta * K * K^T.
Args:
q (torch.Tensor):
The query tensor of shape `[B, T, H, K]`.
k (torch.Tensor):
The key tensor of shape `[B, T, H, K]`.
beta (torch.Tensor):
The beta tensor of shape `[B, T, H]`.
gk (torch.Tensor):
The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
scale (float):
Scale applied to the query-key products. Default: `None`.
cu_seqlens (torch.Tensor):
The cumulative sequence lengths of the input tensor.
Default: None
chunk_indices (torch.Tensor):
Precomputed chunk indices for `cu_seqlens`. Default: `None`.
chunk_size (int):
The chunk size. Default: 64.
output_dtype (torch.dtype):
@@ -426,6 +431,7 @@ def chunk_kda_scaled_dot_kkt_fwd(
Returns:
beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
"""
B, T, H, K = k.shape
assert K <= 256
@@ -1318,16 +1324,15 @@ def fused_kda_gate(
safe_gate: bool = False,
lower_bound: float | None = -5.0,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter (softplus branch only)
threshold: softplus threshold parameter (softplus branch only)
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
lower_bound: floor for the safe_gate branch (default -5.0)
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter (softplus branch only)
threshold: softplus threshold parameter (softplus branch only)
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
lower_bound: floor for the safe_gate branch (default -5.0)
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+4 -2
View File
@@ -130,7 +130,8 @@ def _resolve_kda_prefill_backend(
"""Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on
SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate) or the Triton
``chunk_kda_with_fused_gate`` path. ``backend`` comes from
``additional_config.kda_prefill_backend`` (auto / triton / flashkda)."""
``additional_config.kda_prefill_backend`` (auto / triton / flashkda).
"""
if backend not in ("auto", "triton", "flashkda"):
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
capability = current_platform.get_device_capability()
@@ -368,7 +369,8 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
bounded gate ``lower_bound * sigmoid(exp(A_log) * (g + dt_bias))``,
matching ``chunk_kda_with_fused_gate(..., safe_gate=True)``. Writes the
attention output into ``out`` (a workspace buffer when ``None``) and
returns ``(out, final_state)``."""
returns ``(out, final_state)``.
"""
assert self._flashkda_buffer_specs is not None
final_state, workspace, workspace_out = (
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
+2 -1
View File
@@ -720,7 +720,8 @@ class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
(image token repeat, video frame/timestamp structure) is owned by vLLM's
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
the replacement content and the placeholder scan validates against
exactly that."""
exactly that.
"""
def _hf_processor_applies_updates(
self,
@@ -138,7 +138,8 @@ def _gather_workspace_shapes(
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
"""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
@@ -156,7 +157,8 @@ def kv_cache_as_quant_view(
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation."""
DeepGEMM, from the 3D indexer kv-cache allocation.
"""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
@@ -31,7 +31,8 @@ def _fused_eh_norm_kernel(
):
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
"""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
@@ -116,6 +116,7 @@ def fwht128_quant_fp8(q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
Returns:
(q_fp8 ``[rows, 128]`` float8_e4m3fn, scale ``[rows, 1]`` float32).
"""
assert q.ndim == 2 and q.shape[1] == 128, q.shape
assert q.dtype == torch.bfloat16
@@ -162,7 +163,8 @@ def _kpool_softmax_rotate_write_cache_kernel(
BLOCK_D: tl.constexpr,
):
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
"""
row = tl.program_id(0)
do_write = True
if HAS_WRITE_MASK:
@@ -276,6 +278,13 @@ def kpool_compress_and_write_cache(
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
loc: ``[n_pools]`` int64 — flat physical slot per pool.
pool_size: Number of tokens compressed into one cache entry.
head_dim: Indexer head dimension.
write_mask: ``[n_pools]`` bool — pools to write, or None for all.
round_scale: Round each fp8 scale down to a power of two.
return_compressed: Also return the compressed K and scales.
write_cache: Write the compressed result into ``kv_cache``.
"""
assert slot_k.ndim == 3
assert slot_score.shape == slot_k.shape
@@ -642,6 +651,10 @@ def kpool_decode_update_and_maybe_write_cache_batched(
ape: ``[pool_size, head_dim]`` fp32.
slot_mapping: ``[num_requests, next_n]`` int32.
positions: ``[num_requests, next_n]`` int32.
pool_size: Number of tokens compressed into one cache entry.
head_dim: Indexer head dimension.
round_scale: Round each fp8 scale down to a power of two.
"""
num_requests, next_n = key.shape[0], key.shape[1]
if num_requests == 0 or next_n == 0:
@@ -590,8 +590,7 @@ def fused_recurrent_gated_delta_rule(
num_accepted_tokens: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
r"""
Args:
r"""Args:
q (torch.Tensor):
queries of shape `[B, T, H, K]`.
k (torch.Tensor):
@@ -653,6 +652,7 @@ def fused_recurrent_gated_delta_rule(
initial_state=h0,
cu_seqlens=cu_seqlens
)
"""
if cu_seqlens is not None and q.shape[0] != 1:
raise ValueError(
+17 -12
View File
@@ -416,19 +416,24 @@ def chunk_kda_scaled_dot_kkt_fwd(
chunk_size: int = FLA_CHUNK_SIZE,
output_dtype: torch.dtype = torch.float32,
) -> tuple[torch.Tensor, torch.Tensor]:
r"""
Compute beta * K * K^T.
r"""Compute beta * K * K^T.
Args:
q (torch.Tensor):
The query tensor of shape `[B, T, H, K]`.
k (torch.Tensor):
The key tensor of shape `[B, T, H, K]`.
beta (torch.Tensor):
The beta tensor of shape `[B, T, H]`.
gk (torch.Tensor):
The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
scale (float):
Scale applied to the query-key products. Default: `None`.
cu_seqlens (torch.Tensor):
The cumulative sequence lengths of the input tensor.
Default: None
chunk_indices (torch.Tensor):
Precomputed chunk indices for `cu_seqlens`. Default: `None`.
chunk_size (int):
The chunk size. Default: 64.
output_dtype (torch.dtype):
@@ -436,6 +441,7 @@ def chunk_kda_scaled_dot_kkt_fwd(
Returns:
beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
"""
B, T, H, K = k.shape
assert K <= 256
@@ -1328,16 +1334,15 @@ def fused_kda_gate(
safe_gate: bool = False,
lower_bound: float | None = -5.0,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter (softplus branch only)
threshold: softplus threshold parameter (softplus branch only)
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
lower_bound: floor for the safe_gate branch (default -5.0)
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter (softplus branch only)
threshold: softplus threshold parameter (softplus branch only)
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
lower_bound: floor for the safe_gate branch (default -5.0)
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+3
View File
@@ -67,6 +67,7 @@ def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]:
Raises:
ValueError: If ``indexer_types`` has the wrong length or an unknown
entry, or if ``index_topk_freq`` is not a positive integer.
"""
if not hasattr(config, "index_topk"):
return set()
@@ -115,6 +116,7 @@ def is_skip_topk_indexer_weight(weight_name: str, skip_topk_layers: set[int]) ->
Returns:
True when the weight is an indexer weight of a layer that has no
indexer module and therefore must be dropped.
"""
if ".indexer." not in weight_name or not skip_topk_layers:
return False
@@ -557,6 +559,7 @@ class HYV4MLAAttention(nn.Module):
The backend class to bind, or None when no sink-capable backend is
available; the caller then loads the sink weight but disables the
bias.
"""
head_size = self.kv_lora_rank + self.qk_rope_head_dim
dtype = torch.get_default_dtype()
@@ -99,6 +99,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
Raises:
ValueError: If the dtype is not float32 or the shape is not
``(num_heads,)``.
"""
if sinks is None:
return
@@ -134,6 +135,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
Raises:
ValueError: If the sink and query head layouts disagree, or if they
live on different devices.
"""
sinks = self.sinks
if sinks is None:
+4
View File
@@ -107,6 +107,7 @@ class HYV4HCPreLayer(nn.Module):
Returns:
A tuple of the pre-gated reduction ``[num_tokens, d]`` and the post
gates ``[num_tokens, hc]`` consumed by `HYV4HCPostLayer`.
"""
if self.hpc_op is not None:
return self.hpc_op(x)
@@ -186,6 +187,7 @@ class HYV4HCPostLayer(nn.Module):
Returns:
The updated residual channels ``[num_tokens, hc, d]``.
"""
if self.hpc_op is not None:
return self.hpc_op(x, residual, post)
@@ -272,6 +274,7 @@ class HYV4HCHeadLayer(nn.Module):
Returns:
The merged hidden state ``[num_tokens, d]``.
"""
if self.hpc_op is not None:
return self.hpc_op(x)
@@ -367,6 +370,7 @@ class HYV4HCLayer(nn.Module):
A tuple of the reduced hidden states ``[num_tokens, d]``, the post
gates ``[num_tokens, hc]`` (``None`` when iHC is disabled) and the
residual (the untouched input).
"""
if not self.enable_ihc:
return hidden_states, None, hidden_states
+1 -1
View File
@@ -189,7 +189,7 @@ class HYV4DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, None]:
"""iHC forward: each sub-block reduces and re-scatters the channels."""
"""IHC forward: each sub-block reduces and re-scatters the channels."""
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
hidden_states, post_gates, residual = self.hc_attn_layer.pre(hidden_states)
hidden_states = self.input_layernorm(hidden_states)
+4
View File
@@ -106,6 +106,7 @@ def _resolve_fused_expert_param(
Returns:
The matching draft parameter name, or None when there is none.
"""
if not ckpt_suffix:
return param_base if param_base in params_dict else None
@@ -164,6 +165,7 @@ def _create_mtp_quant_config(
Returns:
The quantization config to use for the MTP layers, or None when the MTP
layers are unquantized.
"""
mtp_quant_algo = getattr(hf_config, "mtp_quant_algo", None)
@@ -241,6 +243,7 @@ def _remap_mtp_quant_exclusions(
Returns:
A shallow copy with the translated exclusions, or the input unchanged.
"""
if quant_config is None:
return None
@@ -671,6 +674,7 @@ class HYV4MTP(nn.Module):
Returns:
True when the weight was consumed (even if this rank holds none of
the addressed experts).
"""
base = name.split(".experts.")[0]
for ckpt_proj, tag in (
@@ -37,6 +37,7 @@ class InklingLogitsProcessor(LogitsProcessor):
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
logits_mup_width_multiplier: muP width divisor for the final logits;
``None`` or ``0`` disables it.
"""
def __init__(
+2 -1
View File
@@ -72,7 +72,8 @@ def _sconv_add_norm(
ROCm uses the portable collective path. The Lamport P2P implementation in
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
publication semantics which cannot be linked into a gfx950 HSACO."""
publication semantics which cannot be linked into a gfx950 HSACO.
"""
norm_w = norm.weight if norm is not None else None
eps = norm.variance_epsilon if norm is not None else 0.0
+6 -3
View File
@@ -192,7 +192,8 @@ def inkling_gate_select(
class InklingGate(nn.Module):
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
global scale (the served checkpoint's only configuration)."""
global scale (the served checkpoint's only configuration).
"""
def __init__(
self,
@@ -246,7 +247,8 @@ class InklingGate(nn.Module):
self, gating_output: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
are the routed top-k; the S trailing entries are the sink gammas."""
are the routed top-k; the S trailing entries are the sink gammas.
"""
return inkling_gate_select(
gating_output,
self.n_total_experts,
@@ -267,7 +269,8 @@ class InklingGate(nn.Module):
def _inkling_moe_ep_size() -> int:
"""EP size the MoERunner layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled)."""
expert parallelism is enabled).
"""
parallel_config = get_current_vllm_config().parallel_config
if not parallel_config.enable_expert_parallel:
return 1
+6 -3
View File
@@ -166,7 +166,8 @@ class InklingMultiTokenPredictor(nn.Module):
) -> torch.Tensor:
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
the target's tower embeddings scattered in unnormed (the backbone
convention — MM embeds are merged after embed_norm)."""
convention — MM embeds are merged after embed_norm).
"""
norm = self.backbone_embed_norm
embeds = embed_rmsnorm(
input_ids,
@@ -195,7 +196,8 @@ class InklingMultiTokenPredictor(nn.Module):
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
launch: embedding row gather + the backbone embed_norm + the depth
embed_norm chain on one side, hidden_norm on the other, written
straight into the cat buffer."""
straight into the cat buffer.
"""
hidden_w = layer.hidden_norm.weight
embed_w = layer.embed_norm.weight
eps = layer.hidden_norm.variance_epsilon
@@ -327,7 +329,8 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
reduction — no full-vocab logits all-gather. The muP divisor is a
positive scalar, so the argmax is invariant and the scaling is
skipped entirely."""
skipped entirely.
"""
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+4 -2
View File
@@ -238,7 +238,8 @@ def embed_rmsnorm(
embedding-table gather.
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
first decoder layer's pre-attention norm) as a second output, still one
launch. Bit-exact vs the unfused module sequence."""
launch. Bit-exact vs the unfused module sequence.
"""
ids = input_ids.view(-1)
(T,) = ids.shape
n = embed_table.shape[1]
@@ -332,7 +333,8 @@ def embed_dual_rmsnorm_cat(
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
unfused sequence). The concat copies collapse into direct writes."""
unfused sequence). The concat copies collapse into direct writes.
"""
T, n = hidden.shape
if embeds is not None:
assert embeds.shape == hidden.shape
@@ -37,6 +37,7 @@ class InklingLogitsProcessor(LogitsProcessor):
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
logits_mup_width_multiplier: muP width divisor for the final logits;
``None`` or ``0`` disables it.
"""
def __init__(
+2 -1
View File
@@ -75,7 +75,8 @@ def _sconv_add_norm(
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
The Lamport path performs reduce-scatter + shard sconv + all-gather +
residual add + norm. The NCCL path handles unsupported configurations."""
residual add + norm. The NCCL path handles unsupported configurations.
"""
attn_metadata = get_forward_context().attn_metadata
m = (
attn_metadata.get(sconv.owner.prefix)
+6 -3
View File
@@ -191,7 +191,8 @@ def inkling_gate_select(
class InklingGate(nn.Module):
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
global scale (the served checkpoint's only configuration)."""
global scale (the served checkpoint's only configuration).
"""
def __init__(
self,
@@ -243,7 +244,8 @@ class InklingGate(nn.Module):
self, gating_output: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
are the routed top-k; the S trailing entries are the sink gammas."""
are the routed top-k; the S trailing entries are the sink gammas.
"""
return inkling_gate_select(
gating_output,
self.n_total_experts,
@@ -264,7 +266,8 @@ class InklingGate(nn.Module):
def _inkling_moe_ep_size() -> int:
"""EP size the FusedMoE layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled)."""
expert parallelism is enabled).
"""
parallel_config = get_current_vllm_config().parallel_config
if not parallel_config.enable_expert_parallel:
return 1
+6 -3
View File
@@ -166,7 +166,8 @@ class InklingMultiTokenPredictor(nn.Module):
) -> torch.Tensor:
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
the target's tower embeddings scattered in unnormed (the backbone
convention — MM embeds are merged after embed_norm)."""
convention — MM embeds are merged after embed_norm).
"""
norm = self.backbone_embed_norm
embeds = embed_rmsnorm(
input_ids,
@@ -195,7 +196,8 @@ class InklingMultiTokenPredictor(nn.Module):
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
launch: embedding row gather + the backbone embed_norm + the depth
embed_norm chain on one side, hidden_norm on the other, written
straight into the cat buffer."""
straight into the cat buffer.
"""
hidden_w = layer.hidden_norm.weight
embed_w = layer.embed_norm.weight
eps = layer.hidden_norm.variance_epsilon
@@ -327,7 +329,8 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
reduction — no full-vocab logits all-gather. The muP divisor is a
positive scalar, so the argmax is invariant and the scaling is
skipped entirely."""
skipped entirely.
"""
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+4 -2
View File
@@ -60,7 +60,8 @@ def dmel_embed_sum_norm(
eps: float,
) -> torch.Tensor:
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
[T, NB, D] intermediate)."""
[T, NB, D] intermediate).
"""
T, nb = dmel_idx.shape
D = weight.shape[1]
assert weight.shape[0] % nb == 0
@@ -151,7 +152,8 @@ def rmsnorm_gelu(
) -> torch.Tensor:
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
comes back as ``fold_timespace_to_depth(result, *fold)``."""
comes back as ``fold_timespace_to_depth(result, *fold)``.
"""
D = x.shape[-1]
flat = x.reshape(-1, D)
assert flat.stride(1) == 1 and flat.stride(0) == D
+4 -2
View File
@@ -238,7 +238,8 @@ def embed_rmsnorm(
embedding-table gather.
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
first decoder layer's pre-attention norm) as a second output, still one
launch. Bit-exact vs the unfused module sequence."""
launch. Bit-exact vs the unfused module sequence.
"""
ids = input_ids.view(-1)
(T,) = ids.shape
n = embed_table.shape[1]
@@ -332,7 +333,8 @@ def embed_dual_rmsnorm_cat(
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
unfused sequence). The concat copies collapse into direct writes."""
unfused sequence). The concat copies collapse into direct writes.
"""
T, n = hidden.shape
if embeds is not None:
assert embeds.shape == hidden.shape
+1 -3
View File
@@ -61,9 +61,7 @@ class ROCmLatentMoERunner(MoERunner):
shared_output: torch.Tensor,
trunc_size: int | None,
) -> torch.Tensor:
"""
Tier 2: column-parallel up-projection folded into the final reduce.
"""
"""Tier 2: column-parallel up-projection folded into the final reduce."""
if not self._logged_sharded_tail:
self._logged_sharded_tail = True
logger.info_once(
+1 -3
View File
@@ -317,9 +317,7 @@ class KimiMoE(nn.Module):
class KimiMLAAttention(nn.Module):
"""
Main reference: DeepseekV2 vllm Implementation
"""
"""Main reference: DeepseekV2 vllm Implementation."""
def __init__(
self,
+9
View File
@@ -213,9 +213,17 @@ def fused_kda_chunk(
Args:
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
w: chunk-local WY representation of the gated keys.
u: chunk-local WY representation of the values.
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
aqk: intra-chunk query-key attention, ``[chunks, H, 64, 64]``.
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
scale: scale applied to the query-key products.
cu_seqlens: int32 cumulative sequence lengths.
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
output_final_state: whether to return the final recurrent state.
chunk_offsets: int32 per-sequence first chunk index.
checkpoint_state: destination for the mid-prefill state snapshots,
fp32 ``[rows, H, 128, 128]``. Without
``checkpoint_state_indices`` it is a staging buffer indexed by
@@ -239,6 +247,7 @@ def fused_kda_chunk(
has_initial_state: bool ``[N]``; a false entry starts that sequence
from a zero state and its cache row is read only, never before the
walk writes it.
"""
if state_cache is not None:
if initial_state is not None or output_final_state:
@@ -52,6 +52,22 @@ def chunk_kda_prefill(
"""Run chunk KDA from raw gate and beta projections.
Args:
q: query tensor, ``[1, T, H, K]``.
k: key tensor, ``[1, T, H, K]``.
v: value tensor, ``[1, T, H, V]``.
raw_g: raw gate projection, before the activation.
raw_beta: raw beta projection, before the activation.
A_log: log of the per-head gate decay.
g_bias: optional per-head gate bias.
scale: scale applied to the query-key products. Defaults to
``k.shape[-1] ** -0.5``.
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
output_final_state: whether to return the final recurrent state.
lower_bound: optional floor applied to the gate.
use_qk_l2norm_in_kernel: fold the q/k L2 norm into the kernel.
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
@@ -71,6 +87,7 @@ def chunk_kda_prefill(
Returns:
The output and, when requested, the final recurrent state.
"""
if scale is None:
scale = k.shape[-1] ** -0.5
+6 -7
View File
@@ -910,13 +910,12 @@ def fused_kda_gate(
threshold: float = 20.0,
lower_bound: float | None = None,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+1 -2
View File
@@ -59,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
USE_SAFE_GATE: tl.constexpr,
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
):
"""
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
This kernel:
@@ -151,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
chunk_size: int = 64,
sub_chunk_size: int = 16,
) -> None:
"""
Token-parallel implementation: each token gets its own thread block.
"""Token-parallel implementation: each token gets its own thread block.
Supports both fixed-length and variable-length sequences.
Reduces wasted computation on padding.
@@ -164,10 +163,12 @@ def chunk_kda_fwd_intra_token_parallel(
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
beta: [B, T, HV]
Aqk: [B, T, HV, BT] output tensor to write to
cu_seqlens: cumulative sequence lengths for variable-length input
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
scale: attention scale
chunk_size: BT (default 64)
sub_chunk_size: BC (default 16)
"""
B, T, H, K, HV = *q.shape, gk.shape[2]
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
@@ -275,9 +275,7 @@ class LatentMoERunner(MoERunner):
shared_output: torch.Tensor,
trunc_size: int | None,
) -> torch.Tensor:
"""
Tier 2: column-parallel up-projection folded into the final reduce.
"""
"""Tier 2: column-parallel up-projection folded into the final reduce."""
transform = self.routed_output_transform
assert transform is not None
@@ -751,6 +751,7 @@ def run_kda_projection_overlap(
Returns:
The Q/K/V/G projection, F_B output, and beta projection.
"""
num_tokens = hidden_states.shape[0]
qkvg_weight = packed_weight[:_KDA_QKVG_SIZE]
+2 -1
View File
@@ -745,7 +745,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
slot_mapping: torch.Tensor,
) -> torch.Tensor:
"""Fused decode query-concat + latent cache insert, dispatched by cache
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
dtype (same policy as prefill: fp8 cache -> fp8 query).
"""
if self.kv_cache_dtype == "fp8_ds_mla":
cache = self.kv_cache
if cache.dtype != torch.uint8:
+2
View File
@@ -339,6 +339,7 @@ class KimiRoutedOutputTransform(nn.Module):
residual: Optional tensor of the up-projection's output shape to
accumulate into. It is consumed in the GEMM's beta-add
epilogue, so adding it costs no extra kernel.
"""
if self.norm is not None:
hidden_states = self.norm(hidden_states)
@@ -796,6 +797,7 @@ class KimiMoE(nn.Module):
``router_output`` holds the grouped top-k weights and ``topk_ids``
the selected experts; otherwise ``router_output`` holds the raw gate
logits and ``topk_ids`` is ``None``.
"""
def _router(
@@ -59,7 +59,6 @@ def _mapping(
def validate_shape(*, tp_size: int, latent_dim: int, hidden_dim: int) -> None:
"""Validate constraints imposed by the fused collective mapping."""
if tp_size <= 0 or latent_dim <= 0 or hidden_dim <= 0:
raise ValueError("collective dimensions must be positive")
if hidden_dim % tp_size:
@@ -86,7 +85,6 @@ def _select_routed_schedule(
max_m: int,
) -> tuple[int, int]:
"""Match upstream MNNVL's one-cluster-per-token occupancy policy."""
_, cluster_ctas, threads, _ = _mapping(tp_size, latent_dim, hidden_dim)
sm_count = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
@@ -886,7 +884,6 @@ def compile_kernel(
top_k: int = 0,
) -> None:
"""Compile the rank/M specialization without retaining caller tensors."""
key = _compile_key(
rank,
tp_size,
@@ -79,7 +79,6 @@ def validate_configuration(
b_prime_stages: int,
) -> None:
"""Validate constraints imposed by this BF16 SM100 GEMM."""
if latent_dim <= 0 or shard_dim <= 0:
raise ValueError("GEMM K and N must be positive")
if latent_dim % 8 or shard_dim % 8:
@@ -52,7 +52,6 @@ def to_cute_dynamic_m(
mode is symbolic, so changing M within an Op's capacity reuses the same
compiled kernel.
"""
return to_cute(tensor, assumed_align).mark_compact_shape_dynamic(
mode=mode,
stride_order=tensor.dim_order(),
@@ -98,7 +97,6 @@ def load_global_u32x4(
side-effecting prevents loop-invariant motion and common-subexpression
elimination across polling iterations.
"""
address = pointer.toint(loc=loc, ip=ip)
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
out = llvm.inline_asm(
@@ -194,7 +192,6 @@ def finalize_top16_bf16(
ip=None,
):
"""Finalize one Kimi K3 top-16 vector with wide metadata loads."""
addresses = [
pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
for pointer in (gemm2_vector, route_indices, route_weights)
@@ -229,7 +226,6 @@ def store_global_u32x4(
ip=None,
) -> None:
"""Store four packed words to an ordinary or NVLS multicast global VA."""
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
opcode = "st.volatile.global.v4.u32" if volatile else "st.global.v4.u32"
llvm.inline_asm(
@@ -294,7 +290,6 @@ def store_global_u32(
@dsl_user_op
def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
"""Publish eight BF16 values through an NVLS multicast mapping."""
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
llvm.inline_asm(
None,
@@ -312,7 +307,6 @@ def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
@dsl_user_op
def store_lamport_sentinel_128(pointer: cute.Pointer, *, loc=None, ip=None) -> None:
"""Reset one Lamport fragment to four FP32 negative-zero bit patterns."""
address = pointer.toint(loc=loc, ip=ip)
value = Uint32(NEG_ZERO_F32_BITS).ir_value(loc=loc, ip=ip)
llvm.inline_asm(
@@ -333,7 +327,6 @@ def red_async_release_gpu_add_u32(
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
) -> None:
"""The exact SM100 arrival primitive used by upstream LamportFlags."""
address = pointer.toint(loc=loc, ip=ip)
llvm.inline_asm(
None,
@@ -378,7 +371,6 @@ def map_shared_to_peer(
ip=None,
) -> Int32:
"""Map a local shared-memory slot to the same slot in a peer CTA."""
smem_address = smem_ptr.toint(loc=loc, ip=ip).ir_value()
return Int32(
llvm.inline_asm(
@@ -422,7 +414,6 @@ def store_shared_cluster_f32(
@dsl_user_op
def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
"""Load two aligned FP32 DSM partials from local shared memory."""
address = Int32(pointer.toint(loc=loc, ip=ip))
out = llvm.inline_asm(
llvm.StructType.get_literal([T.f32()] * 2),
@@ -443,7 +434,6 @@ def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
@dsl_user_op
def load_shared_f32x4(pointer: cute.Pointer, *, loc=None, ip=None):
"""Load four aligned FP32 DSM partials from local shared memory."""
address = Int32(pointer.toint(loc=loc, ip=ip))
out = llvm.inline_asm(
llvm.StructType.get_literal([T.f32()] * 4),
@@ -530,7 +520,6 @@ def sanitize_negative_zero_u32x2(packed):
@cute.jit
def sanitize_negative_zero(packed):
"""Turn real BF16 -0 into +0 so it cannot equal the empty sentinel."""
result = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
for i in cutlass.range_constexpr(4):
word = packed[i]
@@ -547,7 +536,6 @@ def sanitize_negative_zero(packed):
@cute.jit
def fragment_is_dirty(packed):
"""Bit-exact upstream sentinel check: one comparison per 32-bit word."""
dirty = packed[0] == Uint32(NEG_ZERO_F32_BITS)
for i in cutlass.range_constexpr(1, 4):
dirty = dirty | (packed[i] == Uint32(NEG_ZERO_F32_BITS))
@@ -564,7 +552,6 @@ def warp_sum_specialized(
last_warp_mask: cutlass.Constexpr[int],
) -> Float32:
"""Warp sum supporting a compile-time partial final warp."""
if warp_idx == Int32(warps - 1) and cutlass.const_expr(last_warp_lanes < 32):
for offset in cutlass.range_constexpr(16, 0, -1):
# range_constexpr does not provide powers-of-two stepping.
+7 -10
View File
@@ -539,9 +539,7 @@ def fused_kda_gate_chunk_cumsum(
)
B, T, H, D = raw_g.shape
if raw_beta.shape != (B, T, H):
raise ValueError(
f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}"
)
raise ValueError(f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}")
if chunk_indices is None and cu_seqlens is not None:
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
@@ -901,13 +899,12 @@ def fused_kda_gate(
threshold: float = 20.0,
lower_bound: float | None = None,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+200 -98
View File
@@ -23,18 +23,20 @@ from .chunk_intra_token_parallel import chunk_kda_fwd_intra_token_parallel
################################################################################
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({'BK': BK}, num_warps=num_warps)
triton.Config({"BK": BK}, num_warps=num_warps)
for BK in [32, 64]
for num_warps in [1, 2, 4]
],
key=["H", "HV", "K", "BC"],
)
@triton.jit(do_not_specialize=['T'])
@triton.jit(do_not_specialize=["T"])
def chunk_kda_fwd_kernel_inter_solve_fused(
q,
k,
@@ -57,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
USE_SAFE_GATE: tl.constexpr,
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
):
"""
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
This kernel:
@@ -74,8 +75,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
@@ -122,21 +129,31 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
p_k0 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
p_g0 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
p_k0 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
p_g0 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
if i_tc1 < T:
p_q1 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_k1 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_g1 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_q1 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_k1 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_g1 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn1 = tl.load(g + i_tc1 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
# [BC, BK]
b_gqn = tl.where(m_tc1[:, None], exp2(b_g1 - b_gn1[None, :]), 0)
# [BK, BC]
@@ -146,15 +163,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
if i_tc2 < T:
p_q2 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_k2 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_g2 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_q2 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_k2 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_g2 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn2 = tl.load(g + i_tc2 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
)
# [BC, BK]
b_gqn2 = tl.where(m_tc2[:, None], exp2(b_g2 - b_gn2[None, :]), 0)
b_qg2 = b_q2 * b_gqn2
@@ -170,15 +195,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk21 += tl.dot(b_kg2, b_kgt)
if i_tc3 < T:
p_q3 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_k3 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_g3 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_q3 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_k3 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_g3 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn3 = tl.load(g + i_tc3 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
)
# [BC, BK]
b_gqn3 = tl.where(m_tc3[:, None], exp2(b_g3 - b_gn3[None, :]), 0)
b_qg3 = b_q3 * b_gqn3
@@ -203,40 +236,78 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
# save off-diagonal Aqk blocks and prepare Akk
################################################################################
if i_tc1 < T:
p_Aqk10 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk10 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
tl.store(
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b1 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,))
p_b1 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
)
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
b_Akk10 = b_Akk10 * b_b1[:, None]
if i_tc2 < T:
p_Aqk20 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Aqk21 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk20 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Aqk21 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b2 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,))
p_b2 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
)
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
b_Akk20 = b_Akk20 * b_b2[:, None]
b_Akk21 = b_Akk21 * b_b2[:, None]
if i_tc3 < T:
p_Aqk30 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Aqk31 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
p_Aqk32 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk30 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Aqk31 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Aqk32 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b3 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,))
p_b3 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
)
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
b_Akk30 = b_Akk30 * b_b3[:, None]
b_Akk31 = b_Akk31 * b_b3[:, None]
b_Akk32 = b_Akk32 * b_b3[:, None]
p_Akk00 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc0, 0), (BC, BC), (1, 0))
p_Akk11 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc1, 0), (BC, BC), (1, 0))
p_Akk22 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Akk33 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Akk00 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
@@ -256,25 +327,25 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Ai33 = -tl.where(m_A, b_Ai33, 0)
for i in range(2, min(BC, T - i_tc0)):
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a00 = tl.where(o_i < i, b_a00, 0.)
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a00 = tl.where(o_i < i, b_a00, 0.0)
b_a00 += tl.sum(b_a00[:, None] * b_Ai00, 0)
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
for i in range(BC + 2, min(2*BC, T - i_tc0)):
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a11 = tl.where(o_i < i - BC, b_a11, 0.)
for i in range(BC + 2, min(2 * BC, T - i_tc0)):
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a11 = tl.where(o_i < i - BC, b_a11, 0.0)
b_a11 += tl.sum(b_a11[:, None] * b_Ai11, 0)
b_Ai11 = tl.where((o_i == i - BC)[:, None], b_a11, b_Ai11)
for i in range(2*BC + 2, min(3*BC, T - i_tc0)):
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a22 = tl.where(o_i < i - 2*BC, b_a22, 0.)
for i in range(2 * BC + 2, min(3 * BC, T - i_tc0)):
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a22 = tl.where(o_i < i - 2 * BC, b_a22, 0.0)
b_a22 += tl.sum(b_a22[:, None] * b_Ai22, 0)
b_Ai22 = tl.where((o_i == i - 2*BC)[:, None], b_a22, b_Ai22)
for i in range(3*BC + 2, min(4*BC, T - i_tc0)):
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a33 = tl.where(o_i < i - 3*BC, b_a33, 0.)
b_Ai22 = tl.where((o_i == i - 2 * BC)[:, None], b_a22, b_Ai22)
for i in range(3 * BC + 2, min(4 * BC, T - i_tc0)):
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a33 = tl.where(o_i < i - 3 * BC, b_a33, 0.0)
b_a33 += tl.sum(b_a33[:, None] * b_Ai33, 0)
b_Ai33 = tl.where((o_i == i - 3*BC)[:, None], b_a33, b_Ai33)
b_Ai33 = tl.where((o_i == i - 3 * BC)[:, None], b_a33, b_Ai33)
b_Ai00 += m_I
b_Ai11 += m_I
@@ -289,53 +360,73 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Ai10 = -tl.dot(
tl.dot(b_Ai11, b_Akk10, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai00,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai21 = -tl.dot(
tl.dot(b_Ai22, b_Akk21, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai11,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai32 = -tl.dot(
tl.dot(b_Ai33, b_Akk32, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai22,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai20 = -tl.dot(
b_Ai22,
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai31 = -tl.dot(
b_Ai33,
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai30 = -tl.dot(
b_Ai33,
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
################################################################################
# store full Akk_inv to Akk
################################################################################
p_Akk00 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
p_Akk10 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
p_Akk11 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
p_Akk20 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Akk21 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
p_Akk22 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 2*BC), (BC, BC), (1, 0))
p_Akk30 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Akk31 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
p_Akk32 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
p_Akk33 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 3*BC), (BC, BC), (1, 0))
p_Akk00 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk10 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
)
p_Akk20 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk21 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
)
p_Akk30 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Akk31 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Akk32 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
)
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@@ -349,18 +440,20 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
for num_warps in [1, 2, 4, 8]
for num_stages in [2, 3, 4]
],
key=['BK', 'NC', 'BT', 'HV'],
key=["BK", "NC", "BT", "HV"],
)
@triton.jit(do_not_specialize=['B', 'T'])
@triton.jit(do_not_specialize=["B", "T"])
def chunk_kda_fwd_kernel_intra_sub_chunk(
q,
k,
@@ -386,8 +479,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
@@ -406,9 +505,9 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
Aqk = Aqk + (bos * HV + i_hv) * BT
Akk = Akk + (bos * HV + i_hv) * BC
p_q = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_k = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_g = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
@@ -418,10 +517,12 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
if USE_GATHER:
b_gn = gather(b_g, tl.full([1, BK], min(BC//2, T - i_ti - 1), dtype=tl.int16), axis=0)
b_gn = gather(
b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0
)
else:
# caculate offset
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV*K + tl.arange(0, BK)
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + tl.arange(0, BK)
b_gn = tl.load(p_gn, mask=tl.arange(0, BK) < K, other=0.0)
b_gn = b_gn[None, :]
@@ -429,8 +530,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
# less than 85 to avoid overflow in exp2
b_gm = (b_g - b_gn).to(tl.float32)
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.)
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.)
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.0)
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.0)
b_kgt = tl.trans(b_k * b_gk)
@@ -445,8 +546,10 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
p_Aqk = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0))
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV*BC, 1), (i_ti, 0), (BC, BC), (1, 0))
p_Aqk = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
)
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@@ -458,8 +561,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_Ai = -b_Akk
for i in range(2, min(BC, T - i_ti)):
b_a = -tl.load(Akk + (i_ti + i) * HV*BC + o_i)
b_a = tl.where(o_i < i, b_a, 0.)
b_a = -tl.load(Akk + (i_ti + i) * HV * BC + o_i)
b_a = tl.where(o_i < i, b_a, 0.0)
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
b_Ai += m_I
@@ -531,8 +634,7 @@ def chunk_kda_fwd_intra(
# Step 2: Fused inter + solve_tril (works for both fixed-len and varlen)
solve_tril_dot_precision = (
"tf32"
if current_platform.is_cuda()
and current_platform.has_device_capability(80)
if current_platform.is_cuda() and current_platform.has_device_capability(80)
else "ieee"
)
grid = (NT, B * HV)
@@ -16,18 +16,20 @@ from vllm.third_party.flash_linear_attention.ops.op import exp2
from vllm.triton_utils import tl, triton
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({'BH': BH}, num_warps=num_warps)
triton.Config({"BH": BH}, num_warps=num_warps)
for BH in [1, 2, 4, 8]
for num_warps in [1, 2, 4, 8]
],
key=["K", "H", "HV"],
)
@triton.jit(do_not_specialize=['T', 'N'])
@triton.jit(do_not_specialize=["T", "N"])
def chunk_kda_fwd_kernel_intra_token_parallel(
q,
k,
@@ -65,7 +67,10 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
left = mid + 1
i_n = left
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
i_t = i_tg - bos
else:
@@ -82,11 +87,11 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
G: tl.constexpr = HV // H
q += bos * H*K
k += bos * H*K
g += bos * HV*K
Aqk += bos * HV*BT
Akk += bos * HV*BC
q += bos * H * K
k += bos * H * K
g += bos * HV * K
Aqk += bos * HV * BT
Akk += bos * HV * BC
beta += bos * HV
BK: tl.constexpr = triton.next_power_of_2(K)
@@ -103,7 +108,9 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
# g: [B, T, HV, K], beta: [B, T, HV]
p_g = tl.make_block_ptr(g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
p_g = tl.make_block_ptr(
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
@@ -111,15 +118,25 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
p_gj = tl.make_block_ptr(g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
p_gj = tl.make_block_ptr(
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
b_Akk = tl.sum(b_k * b_kgj, axis=1) * tl.where(j < i_t, 1.0, 0.0)
tl.store(Aqk + i_t * HV * BT + o_hv * BT + j % BT, b_Aqk.to(Aqk.dtype.element_ty), mask=m_hv)
tl.store(Akk + i_t * HV * BC + o_hv * BC + j - i_ts, b_Akk.to(Akk.dtype.element_ty), mask=m_hv)
tl.store(
Aqk + i_t * HV * BT + o_hv * BT + j % BT,
b_Aqk.to(Aqk.dtype.element_ty),
mask=m_hv,
)
tl.store(
Akk + i_t * HV * BC + o_hv * BC + j - i_ts,
b_Akk.to(Akk.dtype.element_ty),
mask=m_hv,
)
def chunk_kda_fwd_intra_token_parallel(
@@ -134,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
chunk_size: int = 64,
sub_chunk_size: int = 16,
) -> None:
"""
Token-parallel implementation: each token gets its own thread block.
"""Token-parallel implementation: each token gets its own thread block.
Supports both fixed-length and variable-length sequences.
Reduces wasted computation on padding.
@@ -147,17 +163,21 @@ def chunk_kda_fwd_intra_token_parallel(
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
beta: [B, T, HV]
Aqk: [B, T, HV, BT] output tensor to write to
cu_seqlens: cumulative sequence lengths for variable-length input
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
scale: attention scale
chunk_size: BT (default 64)
sub_chunk_size: BC (default 16)
"""
B, T, H, K, HV = *q.shape, gk.shape[2]
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
BT = chunk_size
BC = sub_chunk_size
def grid(meta): return (B * T, triton.cdiv(HV, meta['BH']))
def grid(meta):
return (B * T, triton.cdiv(HV, meta["BH"]))
chunk_kda_fwd_kernel_intra_token_parallel[grid](
q=q,
k=k,
@@ -483,9 +483,7 @@ def fused_recurrent_kda(
)
@triton.jit(
do_not_specialize=["stride_beta_token", "stride_state_indices"]
)
@triton.jit(do_not_specialize=["stride_beta_token", "stride_state_indices"])
def fused_recurrent_kda_packed_decode_kernel(
mixed_qkv,
raw_g,
+2 -1
View File
@@ -334,7 +334,8 @@ def _aiter_moe_fused_shared_experts_enabled(
class MiniMaxM3MoE(nn.Module):
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
expert."""
expert.
"""
def __init__(
self,
+4 -2
View File
@@ -173,7 +173,8 @@ class MiniMaxM3IndexerPrefillMetadata:
@dataclass
class MiniMaxM3IndexerDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens)."""
per-request query length (1, or 1 + num_speculative_tokens).
"""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
@@ -206,7 +207,8 @@ class MiniMaxM3IndexerMetadataBuilder(
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
):
"""Abstract base: shared setup only. The Triton and MSA builders are
parallel subclasses that each own their full ``build`` (no shared code)."""
parallel subclasses that each own their full ``build`` (no shared code).
"""
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
@@ -203,7 +203,8 @@ class MiniMaxM3SparsePrefillMetadata:
@dataclass
class MiniMaxM3SparseDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens)."""
per-request query length (1, or 1 + num_speculative_tokens).
"""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
+4 -2
View File
@@ -85,7 +85,8 @@ class MiniMaxM3IndexerMSAPrefillMetadata:
class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
"""Decode reuses the inherited base ``decode`` field (the Triton decode
metadata); ``prefill_msa`` carries the fmha score plan for the prefill side
(the base ``prefill`` field is unused on this path)."""
(the base ``prefill`` field is unused on this path).
"""
prefill_msa: MiniMaxM3IndexerMSAPrefillMetadata | None = None
# Per-forward view (``[:num_tokens]``) of the builder's persistent unified
@@ -111,7 +112,8 @@ class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
"""Decode metadata is the cudagraph-safe Triton decode metadata; the prefill
fmha plan is built eagerly (prefill batches are not captured)."""
fmha plan is built eagerly (prefill batches are not captured).
"""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
+4 -2
View File
@@ -190,7 +190,8 @@ class MiniMaxM3MLP(nn.Module):
class MiniMaxM3MoE(nn.Module):
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
expert."""
expert.
"""
def __init__(
self,
@@ -782,7 +783,8 @@ class MiniMaxM3DecoderLayer(nn.Module):
@property
def ffn_all_reduce_deferred(self) -> bool:
"""This layer's FFN output is left un-reduced; the caller fuses the
all-reduce into the next RMSNorm."""
all-reduce into the next RMSNorm.
"""
if self.is_moe_layer:
return self.block_sparse_moe.experts.moe_config.skip_final_all_reduce
return not self.mlp.down_proj.reduce_results
-5
View File
@@ -33,7 +33,6 @@ def apply_qsa_rope(
tensor: torch.Tensor,
) -> torch.Tensor:
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
num_tokens, _, head_dim = tensor.shape
rotary_dim = rotary_emb.rotary_dim
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
@@ -67,7 +66,6 @@ def apply_qsa_rmsnorm(
tensor: torch.Tensor,
) -> torch.Tensor:
"""Use vLLM's portable RMSNorm implementation on ROCm."""
return cast(torch.Tensor, norm(tensor))
@@ -153,7 +151,6 @@ class QSAIndexer(nn.Module):
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
qk, _ = self.index_qk_proj(hidden_states)
q_raw, token_k = qk.split(
(
@@ -176,7 +173,6 @@ class QSAIndexer(nn.Module):
first_rope_positions: torch.Tensor,
) -> torch.Tensor:
"""Normalize pooled K and apply the first token's exact group position."""
keys = compressed_keys.reshape(-1, self.index_head_dim)
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
-1, 1, self.index_head_dim
@@ -284,7 +280,6 @@ class QSAIndexer(nn.Module):
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Return fixed-width request-relative token indices padded with ``-1``."""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
@@ -11,5 +11,4 @@ def enable_qwen4_exp_low_latency_gemm(
dtype: torch.dtype,
) -> None:
"""Keep the standard vLLM linear methods on AMD ROCm."""
del module, dtype
-2
View File
@@ -87,7 +87,6 @@ def without_modelopt_fp4(
quant_config: QuantizationConfig | None,
) -> QuantizationConfig | None:
"""Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4."""
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
return None
return quant_config
@@ -103,7 +102,6 @@ def _remap_qsa_cache_scale_name(
that cache directly, so only QSA layers need the final path component
moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers.
"""
scale_suffixes = {
"k_proj.k_scale": "_k_scale",
"k_proj.output_scale": "_k_scale",
-4
View File
@@ -91,7 +91,6 @@ def _remap_quantized_layers(
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
@@ -260,7 +259,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
@@ -271,13 +269,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
-6
View File
@@ -600,7 +600,6 @@ def qsa_mqa_paged(
score_scale: float | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute QSA scores directly from a paged compressed-key cache."""
_validate_mqa(q)
if not q.is_cuda or not HAS_TRITON:
raise RuntimeError("paged QSA scoring requires a GPU and Triton")
@@ -678,7 +677,6 @@ def expand_qsa_block_indices_cuda(
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Expand compressed blocks and compact the causal tail of the open group."""
if not block_indices.is_cuda or not HAS_TRITON:
raise RuntimeError("QSA index expansion requires a GPU and Triton")
if token_topk % compress_ratio:
@@ -738,7 +736,6 @@ def qsa_select_paged_tokens(
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Score, select, and expand QSA indices without host synchronization."""
rows = q.shape[0]
output_width = token_topk + compress_ratio - 1
if out is None:
@@ -829,7 +826,6 @@ def qsa_sparse_paged_attention(
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run sparse GQA directly over paged BF16 K/V caches."""
if not q.is_cuda or not HAS_TRITON:
raise RuntimeError("paged QSA sparse attention requires a GPU and Triton")
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
@@ -971,7 +967,6 @@ def qsa_store_cache_rows(
rows: torch.Tensor,
) -> None:
"""Store fixed-width rows in a QSA cache without boolean indexing."""
if not cache.is_cuda or not HAS_TRITON:
raise RuntimeError("QSA cache stores require a GPU and Triton")
if cache.ndim != 4 or cache.shape[2] != 1:
@@ -1017,7 +1012,6 @@ def qsa_compress_groups_with_ratio(
rope_cache: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pool completed groups from the compressor-state ring and raw token rows."""
if not raw_keys.is_cuda or not HAS_TRITON:
raise RuntimeError("QSA compression requires a GPU and Triton")
rows = token_to_req.numel()
-1
View File
@@ -357,7 +357,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
persistent_buffers = {
"layer_multipliers": self.layer_multipliers,
"ngram_heads_offsets": self.ngram_heads_offsets,
-1
View File
@@ -430,7 +430,6 @@ def qwen4_exp_qsa_with_output(
layer_name: LayerNameType,
) -> None:
"""Run the complete QSA state/update/attend transaction."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
-2
View File
@@ -28,7 +28,6 @@ def compute_ple_shard_overlap(
tp_end: int,
) -> PLEShardOverlap | None:
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
if checkpoint_start < 0 or checkpoint_rows < 0:
raise ValueError("checkpoint shard bounds must be non-negative")
if tp_start < 0 or tp_end < tp_start:
@@ -54,7 +53,6 @@ def copy_ple_embedding_shard_(
tp_end: int,
) -> int:
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
raise ValueError("destination and loaded weight must have matching ranks")
if destination.shape[1:] != loaded_weight.shape[1:]:
@@ -47,7 +47,6 @@ from vllm.v1.kv_cache_interface import (
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
if positions.ndim == 1:
positions = positions.unsqueeze(0).expand(3, -1)
elif positions.ndim != 2 or positions.shape[0] not in (1, 3):
@@ -115,7 +114,6 @@ def circular_qsa_slot_mapping(
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Map each request to its fixed physical block as a circular token ring."""
if compressor_state_size <= 0:
raise ValueError("QSA circular buffer size must be positive")
if block_table.ndim != 2:
@@ -167,7 +165,6 @@ def compressed_qsa_slot_mapping(
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Build boundary-only slots for an ``MLAAttentionSpec`` QSA cache."""
if storage_block_size <= 0 or compress_ratio <= 0:
raise ValueError("QSA block size and compression ratio must be positive")
compressed_positions = torch.div(
@@ -32,7 +32,6 @@ def apply_qsa_rope(
tensor: torch.Tensor,
) -> torch.Tensor:
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
num_tokens, _, head_dim = tensor.shape
rotary_dim = rotary_emb.rotary_dim
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
@@ -245,7 +244,6 @@ class QSAIndexer(nn.Module):
valid-entry count (the attention kernel's loop bound, never a token
index).
"""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
-2
View File
@@ -86,7 +86,6 @@ def without_modelopt_fp4(
quant_config: QuantizationConfig | None,
) -> QuantizationConfig | None:
"""Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4."""
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
return None
return quant_config
@@ -102,7 +101,6 @@ def _remap_qsa_cache_scale_name(
that cache directly, so only QSA layers need the final path component
moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers.
"""
scale_suffixes = {
"k_proj.k_scale": "_k_scale",
"k_proj.output_scale": "_k_scale",
-4
View File
@@ -90,7 +90,6 @@ def _remap_quantized_layers(
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
@@ -250,7 +249,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
@@ -261,13 +259,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
@@ -869,7 +869,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
persistent_buffers = {
"layer_multipliers": self.layer_multipliers,
"ngram_heads_offsets": self.ngram_heads_offsets,
-3
View File
@@ -727,7 +727,6 @@ def warmup_qsa_sparse_paged_attention(
selection_width: int,
) -> tuple[tuple[int, int, int], ...]:
"""Compile every production-reachable split-K/merge specialization."""
head_dim = kv_cache.shape[-1] // 2
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
# An fp8 cache is allocated as uint8 and viewed as e4m3 at attention time.
@@ -870,7 +869,6 @@ def qsa_store_cache_rows(
rows: torch.Tensor,
) -> None:
"""Store fixed-width rows in a QSA cache without boolean indexing."""
if not cache.is_cuda or not HAS_TRITON:
raise RuntimeError("QSA CUDA cache stores require Triton")
if cache.ndim != 4 or cache.shape[2] != 1:
@@ -916,7 +914,6 @@ def qsa_compress_groups_with_ratio(
rope_cache: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pool completed groups from the compressor-state ring and raw token rows."""
if not raw_keys.is_cuda or not HAS_TRITON:
raise RuntimeError("QSA CUDA compression requires Triton")
rows = token_to_req.numel()
@@ -317,7 +317,6 @@ def warmup_qsa_mqa_paged_decode(
max_num_batched_tokens: int,
) -> tuple[tuple[int, int], ...]:
"""Compile every reachable decode specialization without launching it."""
page_size = k_cache.shape[1]
page_table_width = page_table.shape[1]
columns = page_table_width * page_size
@@ -444,7 +443,6 @@ def expand_qsa_block_indices(
out: torch.Tensor,
) -> None:
"""Expand compressed blocks and compact the causal tail of the open group."""
assert token_topk % compress_ratio == 0
block_topk = token_topk // compress_ratio
output_width = token_topk + compress_ratio - 1
@@ -524,8 +522,8 @@ def qsa_select_paged_decode(
compress_ratio: Number of logical tokens represented by a cache row.
decode_query_len: Number of query tokens per request.
block_indices: Compressed-index output buffer.
"""
"""
assert token_topk % compress_ratio == 0
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
assert decode_query_len > 0 and q.shape[0] % decode_query_len == 0
@@ -599,8 +597,8 @@ def qsa_select_paged_prefill(
max_query_len: Maximum number of query tokens in one request.
block_indices: Compressed-index output buffer.
max_seq_len: Longest context length in the batch this step.
"""
"""
assert token_topk % compress_ratio == 0
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
@@ -146,7 +146,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
output_dtype: torch.dtype,
) -> torch.Tensor:
"""Dequantize PLE lookup output."""
return self.ple_embedding.ngram_embedding.dequantize(
embeddings,
output_dtype,