Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1059,7 +1059,8 @@ def _fused_eh_norm_kernel(
|
||||
):
|
||||
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
|
||||
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
|
||||
"""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -1425,7 +1425,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -496,8 +496,7 @@ class DeepSeekV4MTP(nn.Module):
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DeepseekV4 MLA Attention Layer
|
||||
"""
|
||||
"""DeepseekV4 MLA Attention Layer."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
|
||||
@@ -91,7 +91,8 @@ def validate_image_sentinel_ids(tokenizer) -> None:
|
||||
|
||||
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
|
||||
"""Number of LLM tokens the aligner grid occupies (N-layout, including
|
||||
row/align padding)."""
|
||||
row/align padding).
|
||||
"""
|
||||
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
|
||||
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
|
||||
num_tokens = n_llm_h * (n_llm_w + 1) + 2
|
||||
@@ -196,7 +197,8 @@ def load_image(
|
||||
|
||||
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
|
||||
"""Builds the N-layout token types (final order) and the aligner-row order
|
||||
for IMAGE slots."""
|
||||
for IMAGE slots.
|
||||
"""
|
||||
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
|
||||
pad_h = n_llm_h % 2
|
||||
rows = n_llm_h + pad_h
|
||||
@@ -239,7 +241,8 @@ def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
|
||||
|
||||
class DeepseekV4VLImageProcessor:
|
||||
"""Per-image transform (the PIL-input equivalent of the reference
|
||||
``load_image``)."""
|
||||
``load_image``).
|
||||
"""
|
||||
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
|
||||
"""Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
|
||||
preparation.
|
||||
|
||||
- quantize_and_insert_k_cache: quantize bf16 K to UE8M0 FP8 and insert into
|
||||
@@ -61,8 +60,7 @@ def quantize_and_insert_k_kernel(
|
||||
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
|
||||
use_fnuz: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
"""Quantize K tensor and insert into paged K cache.
|
||||
|
||||
K Cache block layout (block_size=64 tokens):
|
||||
- [0, 64*576): Token data, each token has 448 fp8 + 128 bf16
|
||||
@@ -174,8 +172,7 @@ def quantize_and_insert_k_cache(
|
||||
is_ue8m0: bool = True,
|
||||
use_fnuz: bool = False,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
"""Quantize K tensor and insert into paged K cache.
|
||||
|
||||
K Cache block layout (block_size=64 tokens):
|
||||
- First 64 * 576 = 36864 bytes: Token data
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Fused compressor + FP8/MXFP4 UE8M0 quantization + KV cache insert kernels.
|
||||
"""Fused compressor + FP8/MXFP4 UE8M0 quantization + KV cache insert kernels.
|
||||
|
||||
Three specialized kernels:
|
||||
- _fused_kv_compress_norm_rope_insert_sparse_attn:
|
||||
@@ -390,7 +389,7 @@ def _compress_gather_split_sparse_attn(
|
||||
NUM_SPLITS: tl.constexpr,
|
||||
HEAD_TILE: tl.constexpr, # HEAD_SIZE // NUM_SPLITS
|
||||
):
|
||||
"""Stage 1: per-(token, head-split) compress gather, write to fp32 scratch
|
||||
"""Stage 1: per-(token, head-split) compress gather, write to fp32 scratch.
|
||||
|
||||
No-overlap gather (cr>=128) on rows [0, COMPRESS_RATIO)
|
||||
"""
|
||||
@@ -639,7 +638,7 @@ def compress_norm_rope_store_two_stage_triton(
|
||||
num_decode_tokens: int,
|
||||
compress_scratch: torch.Tensor,
|
||||
) -> None:
|
||||
"""Two-stage split compressor dispatch for head=512 cr>=128 (no-overlap)
|
||||
"""Two-stage split compressor dispatch for head=512 cr>=128 (no-overlap).
|
||||
|
||||
Run the occupancy-fanned two-stage split for prefill [num_decodee_tokens:]
|
||||
to fill the CUs, and use the original single-pass launcher
|
||||
|
||||
@@ -312,7 +312,8 @@ class FusedIndexerQRopeQuantTritonKernel(
|
||||
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
|
||||
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
|
||||
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
|
||||
(`SparseMQAIndexer`) is enabled."""
|
||||
(`SparseMQAIndexer`) is enabled.
|
||||
"""
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
return (torch.float32, torch.bfloat16)
|
||||
return (torch.float32,)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
|
||||
"""Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
|
||||
|
||||
Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
|
||||
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
|
||||
@@ -380,6 +379,7 @@ def fused_inv_rope_fp8_quant(
|
||||
Returns:
|
||||
Rotated output in [T, G, D] and its FP8 scales. The scale tensor is
|
||||
empty when quantization is disabled.
|
||||
|
||||
"""
|
||||
from vllm.utils.deep_gemm import get_tma_aligned_size
|
||||
|
||||
|
||||
@@ -263,6 +263,7 @@ def run_dp_sharded_vision_tower(
|
||||
|
||||
Returns:
|
||||
One ``(n_aligner_rows, hidden_size)`` embedding tensor per image.
|
||||
|
||||
"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
@@ -42,7 +42,8 @@ class DeepseekV4CPUCompressor(DeepseekCompressor):
|
||||
weight loading (see
|
||||
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
|
||||
kernel requires fp32/contiguous but the checkpoint loads this weight
|
||||
in bf16, and it never changes after loading."""
|
||||
in bf16, and it never changes after loading.
|
||||
"""
|
||||
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -118,7 +118,8 @@ class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
) -> dict[str, FlashMLASchedMeta | None]:
|
||||
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
|
||||
C++ decode-path concern) -- always return the all-``None`` sentinel
|
||||
the shared base's own CPU branch returns."""
|
||||
the shared base's own CPU branch returns.
|
||||
"""
|
||||
return {
|
||||
_LAYER_TYPE_SWAONLY: None,
|
||||
_LAYER_TYPE_C4A: None,
|
||||
|
||||
@@ -176,13 +176,15 @@ def _fused_indexer_q_rope_quant_cpu(
|
||||
|
||||
class _NoOpEvent:
|
||||
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
|
||||
stream list, these events are constructed but never actually used."""
|
||||
stream list, these events are constructed but never actually used.
|
||||
"""
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
|
||||
"""CPU indexer K-cache descriptor: same fields as the shared base, just
|
||||
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``.
|
||||
"""
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV4CPUIndexerBackend
|
||||
@@ -209,7 +211,8 @@ class DeepseekV4CPUIndexer(DeepseekV4Indexer):
|
||||
compressor run straight-line instead of through
|
||||
``maybe_execute_in_parallel``; the short-context check drops the
|
||||
CUDA-only ``is_current_stream_capturing()`` guard (always False
|
||||
here)."""
|
||||
here).
|
||||
"""
|
||||
compressor = self.compressor
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
@@ -309,7 +312,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
|
||||
``bmm_cpu`` before the FP8 kernel's own
|
||||
``process_weights_after_loading`` VNNI-repacks it in place for
|
||||
row-major reads ``_o_proj`` never does."""
|
||||
row-major reads ``_o_proj`` never does.
|
||||
"""
|
||||
wo_a = self.wo_a
|
||||
quant_method = wo_a.quant_method
|
||||
orig_pwal = quant_method.process_weights_after_loading
|
||||
@@ -400,7 +404,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
|
||||
supported here, so the base method's bf16/per-tensor-fp8 branches
|
||||
are dropped."""
|
||||
are dropped.
|
||||
"""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
@@ -445,7 +450,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""CPU override: two ``RMSNorm`` calls instead of the shared
|
||||
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``).
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
return self.q_norm(qr), None, self.kv_norm(kv)
|
||||
|
||||
@@ -463,7 +469,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
) -> None:
|
||||
"""CPU override: no aux streams, so query-projection+KV-insert, the
|
||||
indexer, and the compressor run straight-line instead of through
|
||||
execute_in_parallel/maybe_execute_in_parallel."""
|
||||
execute_in_parallel/maybe_execute_in_parallel.
|
||||
"""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
@@ -512,7 +519,8 @@ class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: identical body, minus ``@eager_break_during_capture``
|
||||
(this platform never captures a CUDA graph, so it was a no-op)."""
|
||||
(this platform never captures a CUDA graph, so it was a no-op).
|
||||
"""
|
||||
if self.indexer is not None and index_q is not None:
|
||||
assert index_weights is not None
|
||||
# index_q_scale is always None on CPU (FP8-only path).
|
||||
|
||||
@@ -23,6 +23,7 @@ def map_local_to_global_slots_cpu(
|
||||
|
||||
Returns:
|
||||
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
|
||||
|
||||
"""
|
||||
valid = local_indices >= 0
|
||||
safe_local = local_indices.clamp(min=0).to(torch.int64)
|
||||
|
||||
@@ -907,7 +907,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -159,7 +159,8 @@ class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
|
||||
def _realloc_nvfp4_params(self) -> None:
|
||||
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
|
||||
fp8-e4m3 per-16 block scales plus the per-tensor second-level
|
||||
scales (weight_scale_2, input_scale) modelopt exports."""
|
||||
scales (weight_scale_2, input_scale) modelopt exports.
|
||||
"""
|
||||
n_e = self.num_local_experts
|
||||
inter = self.intermediate_size
|
||||
hidden = self.hidden_size
|
||||
|
||||
@@ -56,7 +56,8 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
def _packed_block_span(pool: torch.Tensor) -> int:
|
||||
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
|
||||
block_size for unpacked KV, larger when packed (#44577). Raises if not
|
||||
token-aligned."""
|
||||
token-aligned.
|
||||
"""
|
||||
block_stride = pool.stride(0)
|
||||
token_stride = pool.stride(-2)
|
||||
if block_stride % token_stride != 0:
|
||||
|
||||
@@ -1985,7 +1985,8 @@ class DeepseekV4ForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -518,8 +518,7 @@ class DeepSeekV4MTP(nn.Module):
|
||||
self.finalize_mega_moe_weights()
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -1378,7 +1378,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA)
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -493,8 +493,7 @@ class DeepSeekV4MTP(nn.Module):
|
||||
layer.mtp_block.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -1062,7 +1062,8 @@ class DeepseekV41LLMForCausalLM(
|
||||
@property
|
||||
def token_lookback_depth(self) -> int:
|
||||
"""Tokens before a chunk start the engram hash needs; the model runner
|
||||
passes them as `lookback_token_ids`."""
|
||||
passes them as `lookback_token_ids`.
|
||||
"""
|
||||
engram_hash = self.model.engram_hash
|
||||
return engram_hash.lookback_depth if engram_hash is not None else 0
|
||||
|
||||
@@ -1086,7 +1087,8 @@ class DeepseekV41LLMForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -209,7 +209,8 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
|
||||
self, image_embeds: torch.Tensor, types: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Full image span: aligner rows at IMAGE slots, the learned
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
|
||||
"""
|
||||
types = types.to(image_embeds.device)
|
||||
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
|
||||
dtype = image_embeds.dtype
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DeepseekV4 MLA Attention Layer
|
||||
"""
|
||||
"""DeepseekV4 MLA Attention Layer"""
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -1068,7 +1066,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
def _compressed_kv_cache(self) -> torch.Tensor:
|
||||
"""The compressed-KV cache tensor of this layer's kv source (own
|
||||
cache for kv-source layers)."""
|
||||
cache for kv-source layers).
|
||||
"""
|
||||
if self.is_kv_source:
|
||||
return self.kv_cache
|
||||
assert self.compressed_cache_prefix is not None
|
||||
|
||||
@@ -564,7 +564,8 @@ def _engram_head_shard_weight_loader(
|
||||
param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
"""Load this rank's complete head buckets. ue8m0 scales arrive as
|
||||
float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
|
||||
float8_e8m0fnu; keep the raw bytes (the param stores uint8).
|
||||
"""
|
||||
part_rows = param.shape[0]
|
||||
if loaded_weight.dtype == torch.float8_e8m0fnu:
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
@@ -739,7 +740,8 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
|
||||
bf16, gathered from all shards for this replica's tokens."""
|
||||
bf16, gathered from all shards for this replica's tokens.
|
||||
"""
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
@@ -1000,7 +1002,8 @@ class Engram(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
|
||||
tokens, pre sequence-parallel shard); token_mask: [T], False shuts
|
||||
the gate so those positions pass through untouched."""
|
||||
the gate so those positions pass through untouched.
|
||||
"""
|
||||
kv = self.wkv(self.embed(hash_ids).flatten(-2))
|
||||
num_kv_tokens = hash_ids.shape[0]
|
||||
assert token_mask is None or token_mask.shape == (num_kv_tokens,)
|
||||
|
||||
@@ -82,7 +82,8 @@ def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int:
|
||||
|
||||
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
|
||||
"""Largest aspect-preserving pixel size whose token grid still fits in
|
||||
max_n_token. Returns (best_height, best_width)."""
|
||||
max_n_token. Returns (best_height, best_width).
|
||||
"""
|
||||
r = height / width
|
||||
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
|
||||
max_h_float = max_w_float * r
|
||||
@@ -105,7 +106,8 @@ def safe_resize(
|
||||
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
|
||||
):
|
||||
"""Shrink the pixel size until the image costs at most max_n_token LLM
|
||||
tokens."""
|
||||
tokens.
|
||||
"""
|
||||
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
|
||||
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
|
||||
best_height, best_width = solve_resize_ratio(
|
||||
@@ -171,7 +173,8 @@ def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor:
|
||||
|
||||
class DeepseekV4VLImageProcessor:
|
||||
"""Per-image transform (the PIL-input equivalent of the reference
|
||||
``load_image``)."""
|
||||
``load_image``).
|
||||
"""
|
||||
|
||||
def __init__(self, config: DeepseekV41Config) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
|
||||
"""Triton kernels for DeepseekV4 paged K-cache management and sparse-attention index
|
||||
preparation.
|
||||
|
||||
- quantize_and_insert_k_cache: quantize bf16 K to UE8M0 FP8 and insert into
|
||||
@@ -76,8 +75,7 @@ def quantize_and_insert_k_kernel(
|
||||
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
|
||||
use_fnuz: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
"""Quantize K tensor and insert into paged K cache.
|
||||
|
||||
K Cache block layout (block_size=64 tokens):
|
||||
- [0, 64*576): Token data, each token has 448 fp8 + 128 bf16
|
||||
@@ -251,8 +249,7 @@ def quantize_and_insert_k_cache(
|
||||
use_fnuz: bool = False,
|
||||
bytes_per_token: int = V4_BYTES_PER_TOKEN,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
"""Quantize K tensor and insert into paged K cache.
|
||||
|
||||
``bytes_per_token`` picks the record (see the module header): the V4 one,
|
||||
or V4.1's all-dims MXFP8 one.
|
||||
|
||||
@@ -58,6 +58,7 @@ def fused_save_compress_norm(
|
||||
rms_norm_eps: RMSNorm epsilon.
|
||||
compress_ratio: Group size, either 1 or 2.
|
||||
latent_out: BF16 [tokens, 512], written only at valid group boundaries.
|
||||
|
||||
"""
|
||||
assert compress_ratio in (1, 2)
|
||||
assert kv_score.dtype == torch.float32
|
||||
|
||||
@@ -46,11 +46,13 @@ def indexer_k_norm_rope_store(
|
||||
cos_sin_cache: [max_pos, rope_head_dim] GPT-J layout (cos half, then
|
||||
sin half), from the layer's compress-RoPE instance.
|
||||
rms_norm_weight: [128] k_norm weight.
|
||||
rms_norm_eps: Epsilon of the k_norm RMSNorm.
|
||||
k_cache: uint8 paged indexer cache [num_blocks, block_size, row_bytes].
|
||||
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
|
||||
compress_ratio: group size; keys are emitted at group boundaries.
|
||||
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
|
||||
per-token FP8 with a single fp32 scale.
|
||||
|
||||
"""
|
||||
num_tokens = kv_slot_mapping.numel()
|
||||
assert k_pre.ndim == 2 and k_pre.shape[1] == 128
|
||||
|
||||
@@ -54,7 +54,8 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
def _packed_block_span(pool: torch.Tensor) -> int:
|
||||
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
|
||||
block_size for unpacked KV, larger when packed (#44577). Raises if not
|
||||
token-aligned."""
|
||||
token-aligned.
|
||||
"""
|
||||
block_stride = pool.stride(0)
|
||||
token_stride = pool.stride(-2)
|
||||
if block_stride % token_stride != 0:
|
||||
|
||||
@@ -1179,7 +1179,8 @@ class DeepseekV41LLMForCausalLM(
|
||||
@property
|
||||
def token_lookback_depth(self) -> int:
|
||||
"""Tokens before a chunk start the engram hash needs; the model runner
|
||||
passes them as `lookback_token_ids`."""
|
||||
passes them as `lookback_token_ids`.
|
||||
"""
|
||||
engram_hash = self.model.engram_hash
|
||||
return engram_hash.lookback_depth if engram_hash is not None else 0
|
||||
|
||||
@@ -1203,7 +1204,8 @@ class DeepseekV41LLMForCausalLM(
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-collapse residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
forward(); valid after each target step.
|
||||
"""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -209,7 +209,8 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
|
||||
self, image_embeds: torch.Tensor, types: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Full image span: aligner rows at IMAGE slots, the learned
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
|
||||
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END.
|
||||
"""
|
||||
types = types.to(image_embeds.device)
|
||||
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
|
||||
dtype = image_embeds.dtype
|
||||
|
||||
@@ -194,7 +194,7 @@ class WhisperPositionalEmbedding(nn.Embedding):
|
||||
|
||||
|
||||
class WhisperAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -346,8 +346,7 @@ class WhisperEncoderLayer(nn.Module):
|
||||
rotary_cos: torch.Tensor | None = None,
|
||||
rotary_sin: torch.Tensor | None = None,
|
||||
) -> tuple[Any, ...]:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
hidden_states: Input to the layer of shape `(batch, seq_len,
|
||||
embed_dim)`, or `(total_tokens, embed_dim)` when `cu_seqlens_q`
|
||||
is given.
|
||||
@@ -365,6 +364,7 @@ class WhisperEncoderLayer(nn.Module):
|
||||
Returns:
|
||||
A tuple of the output hidden states, followed by the attention
|
||||
weights if `output_attentions` is `True`.
|
||||
|
||||
"""
|
||||
residual = hidden_states
|
||||
hidden_states = self.self_attn_layer_norm(hidden_states)
|
||||
@@ -435,12 +435,12 @@ class DotsSpeechPreTrainedModel(PreTrainedModel):
|
||||
|
||||
|
||||
class DotsSpeechEncoder(DotsSpeechPreTrainedModel):
|
||||
"""
|
||||
Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
|
||||
"""Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
|
||||
[`WhisperEncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: WhisperConfig
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, config: WhisperConfig):
|
||||
|
||||
@@ -98,7 +98,8 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
@@ -212,6 +213,13 @@ def kpool_compress_and_write_cache(
|
||||
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
|
||||
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
|
||||
loc: ``[n_pools]`` int64 — flat physical slot per pool.
|
||||
pool_size: Number of tokens compressed into one cache entry.
|
||||
head_dim: Indexer head dimension.
|
||||
write_mask: ``[n_pools]`` bool — pools to write, or None for all.
|
||||
round_scale: Round each fp8 scale down to a power of two.
|
||||
return_compressed: Also return the compressed K and scales.
|
||||
write_cache: Write the compressed result into ``kv_cache``.
|
||||
|
||||
"""
|
||||
assert slot_k.ndim == 3
|
||||
assert slot_score.shape == slot_k.shape
|
||||
@@ -592,6 +600,10 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
ape: ``[pool_size, head_dim]`` fp32.
|
||||
slot_mapping: ``[num_requests, next_n]`` int32.
|
||||
positions: ``[num_requests, next_n]`` int32.
|
||||
pool_size: Number of tokens compressed into one cache entry.
|
||||
head_dim: Indexer head dimension.
|
||||
round_scale: Round each fp8 scale down to a power of two.
|
||||
|
||||
"""
|
||||
num_requests, next_n = key.shape[0], key.shape[1]
|
||||
if num_requests == 0 or next_n == 0:
|
||||
|
||||
@@ -564,8 +564,7 @@ def fused_recurrent_gated_delta_rule(
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Args:
|
||||
r"""Args:
|
||||
q (torch.Tensor):
|
||||
queries of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
@@ -627,6 +626,7 @@ def fused_recurrent_gated_delta_rule(
|
||||
initial_state=h0,
|
||||
cu_seqlens=cu_seqlens
|
||||
)
|
||||
|
||||
"""
|
||||
if cu_seqlens is not None and q.shape[0] != 1:
|
||||
raise ValueError(
|
||||
|
||||
+17
-12
@@ -406,19 +406,24 @@ def chunk_kda_scaled_dot_kkt_fwd(
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Compute beta * K * K^T.
|
||||
r"""Compute beta * K * K^T.
|
||||
|
||||
Args:
|
||||
q (torch.Tensor):
|
||||
The query tensor of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
The key tensor of shape `[B, T, H, K]`.
|
||||
beta (torch.Tensor):
|
||||
The beta tensor of shape `[B, T, H]`.
|
||||
gk (torch.Tensor):
|
||||
The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
|
||||
scale (float):
|
||||
Scale applied to the query-key products. Default: `None`.
|
||||
cu_seqlens (torch.Tensor):
|
||||
The cumulative sequence lengths of the input tensor.
|
||||
Default: None
|
||||
chunk_indices (torch.Tensor):
|
||||
Precomputed chunk indices for `cu_seqlens`. Default: `None`.
|
||||
chunk_size (int):
|
||||
The chunk size. Default: 64.
|
||||
output_dtype (torch.dtype):
|
||||
@@ -426,6 +431,7 @@ def chunk_kda_scaled_dot_kkt_fwd(
|
||||
|
||||
Returns:
|
||||
beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
|
||||
|
||||
"""
|
||||
B, T, H, K = k.shape
|
||||
assert K <= 256
|
||||
@@ -1318,16 +1324,15 @@ def fused_kda_gate(
|
||||
safe_gate: bool = False,
|
||||
lower_bound: float | None = -5.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter (softplus branch only)
|
||||
threshold: softplus threshold parameter (softplus branch only)
|
||||
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
|
||||
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
|
||||
lower_bound: floor for the safe_gate branch (default -5.0)
|
||||
return : [..., H, D]
|
||||
"""Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter (softplus branch only)
|
||||
threshold: softplus threshold parameter (softplus branch only)
|
||||
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
|
||||
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
|
||||
lower_bound: floor for the safe_gate branch (default -5.0)
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
|
||||
@@ -130,7 +130,8 @@ def _resolve_kda_prefill_backend(
|
||||
"""Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on
|
||||
SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate) or the Triton
|
||||
``chunk_kda_with_fused_gate`` path. ``backend`` comes from
|
||||
``additional_config.kda_prefill_backend`` (auto / triton / flashkda)."""
|
||||
``additional_config.kda_prefill_backend`` (auto / triton / flashkda).
|
||||
"""
|
||||
if backend not in ("auto", "triton", "flashkda"):
|
||||
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
|
||||
capability = current_platform.get_device_capability()
|
||||
@@ -368,7 +369,8 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
bounded gate ``lower_bound * sigmoid(exp(A_log) * (g + dt_bias))``,
|
||||
matching ``chunk_kda_with_fused_gate(..., safe_gate=True)``. Writes the
|
||||
attention output into ``out`` (a workspace buffer when ``None``) and
|
||||
returns ``(out, final_state)``."""
|
||||
returns ``(out, final_state)``.
|
||||
"""
|
||||
assert self._flashkda_buffer_specs is not None
|
||||
final_state, workspace, workspace_out = (
|
||||
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
|
||||
|
||||
@@ -720,7 +720,8 @@ class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
|
||||
(image token repeat, video frame/timestamp structure) is owned by vLLM's
|
||||
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
|
||||
the replacement content and the placeholder scan validates against
|
||||
exactly that."""
|
||||
exactly that.
|
||||
"""
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
|
||||
@@ -138,7 +138,8 @@ def _gather_workspace_shapes(
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
|
||||
"""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
@@ -156,7 +157,8 @@ def kv_cache_as_quant_view(
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation.
|
||||
"""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
|
||||
@@ -31,7 +31,8 @@ def _fused_eh_norm_kernel(
|
||||
):
|
||||
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
|
||||
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat.
|
||||
"""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -116,6 +116,7 @@ def fwht128_quant_fp8(q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
Returns:
|
||||
(q_fp8 ``[rows, 128]`` float8_e4m3fn, scale ``[rows, 1]`` float32).
|
||||
|
||||
"""
|
||||
assert q.ndim == 2 and q.shape[1] == 128, q.shape
|
||||
assert q.dtype == torch.bfloat16
|
||||
@@ -162,7 +163,8 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
@@ -276,6 +278,13 @@ def kpool_compress_and_write_cache(
|
||||
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
|
||||
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
|
||||
loc: ``[n_pools]`` int64 — flat physical slot per pool.
|
||||
pool_size: Number of tokens compressed into one cache entry.
|
||||
head_dim: Indexer head dimension.
|
||||
write_mask: ``[n_pools]`` bool — pools to write, or None for all.
|
||||
round_scale: Round each fp8 scale down to a power of two.
|
||||
return_compressed: Also return the compressed K and scales.
|
||||
write_cache: Write the compressed result into ``kv_cache``.
|
||||
|
||||
"""
|
||||
assert slot_k.ndim == 3
|
||||
assert slot_score.shape == slot_k.shape
|
||||
@@ -642,6 +651,10 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
ape: ``[pool_size, head_dim]`` fp32.
|
||||
slot_mapping: ``[num_requests, next_n]`` int32.
|
||||
positions: ``[num_requests, next_n]`` int32.
|
||||
pool_size: Number of tokens compressed into one cache entry.
|
||||
head_dim: Indexer head dimension.
|
||||
round_scale: Round each fp8 scale down to a power of two.
|
||||
|
||||
"""
|
||||
num_requests, next_n = key.shape[0], key.shape[1]
|
||||
if num_requests == 0 or next_n == 0:
|
||||
|
||||
@@ -590,8 +590,7 @@ def fused_recurrent_gated_delta_rule(
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Args:
|
||||
r"""Args:
|
||||
q (torch.Tensor):
|
||||
queries of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
@@ -653,6 +652,7 @@ def fused_recurrent_gated_delta_rule(
|
||||
initial_state=h0,
|
||||
cu_seqlens=cu_seqlens
|
||||
)
|
||||
|
||||
"""
|
||||
if cu_seqlens is not None and q.shape[0] != 1:
|
||||
raise ValueError(
|
||||
|
||||
+17
-12
@@ -416,19 +416,24 @@ def chunk_kda_scaled_dot_kkt_fwd(
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Compute beta * K * K^T.
|
||||
r"""Compute beta * K * K^T.
|
||||
|
||||
Args:
|
||||
q (torch.Tensor):
|
||||
The query tensor of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
The key tensor of shape `[B, T, H, K]`.
|
||||
beta (torch.Tensor):
|
||||
The beta tensor of shape `[B, T, H]`.
|
||||
gk (torch.Tensor):
|
||||
The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
|
||||
scale (float):
|
||||
Scale applied to the query-key products. Default: `None`.
|
||||
cu_seqlens (torch.Tensor):
|
||||
The cumulative sequence lengths of the input tensor.
|
||||
Default: None
|
||||
chunk_indices (torch.Tensor):
|
||||
Precomputed chunk indices for `cu_seqlens`. Default: `None`.
|
||||
chunk_size (int):
|
||||
The chunk size. Default: 64.
|
||||
output_dtype (torch.dtype):
|
||||
@@ -436,6 +441,7 @@ def chunk_kda_scaled_dot_kkt_fwd(
|
||||
|
||||
Returns:
|
||||
beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
|
||||
|
||||
"""
|
||||
B, T, H, K = k.shape
|
||||
assert K <= 256
|
||||
@@ -1328,16 +1334,15 @@ def fused_kda_gate(
|
||||
safe_gate: bool = False,
|
||||
lower_bound: float | None = -5.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter (softplus branch only)
|
||||
threshold: softplus threshold parameter (softplus branch only)
|
||||
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
|
||||
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
|
||||
lower_bound: floor for the safe_gate branch (default -5.0)
|
||||
return : [..., H, D]
|
||||
"""Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter (softplus branch only)
|
||||
threshold: softplus threshold parameter (softplus branch only)
|
||||
safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
|
||||
when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
|
||||
lower_bound: floor for the safe_gate branch (default -5.0)
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]:
|
||||
Raises:
|
||||
ValueError: If ``indexer_types`` has the wrong length or an unknown
|
||||
entry, or if ``index_topk_freq`` is not a positive integer.
|
||||
|
||||
"""
|
||||
if not hasattr(config, "index_topk"):
|
||||
return set()
|
||||
@@ -115,6 +116,7 @@ def is_skip_topk_indexer_weight(weight_name: str, skip_topk_layers: set[int]) ->
|
||||
Returns:
|
||||
True when the weight is an indexer weight of a layer that has no
|
||||
indexer module and therefore must be dropped.
|
||||
|
||||
"""
|
||||
if ".indexer." not in weight_name or not skip_topk_layers:
|
||||
return False
|
||||
@@ -557,6 +559,7 @@ class HYV4MLAAttention(nn.Module):
|
||||
The backend class to bind, or None when no sink-capable backend is
|
||||
available; the caller then loads the sink weight but disables the
|
||||
bias.
|
||||
|
||||
"""
|
||||
head_size = self.kv_lora_rank + self.qk_rope_head_dim
|
||||
dtype = torch.get_default_dtype()
|
||||
|
||||
@@ -99,6 +99,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
Raises:
|
||||
ValueError: If the dtype is not float32 or the shape is not
|
||||
``(num_heads,)``.
|
||||
|
||||
"""
|
||||
if sinks is None:
|
||||
return
|
||||
@@ -134,6 +135,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
Raises:
|
||||
ValueError: If the sink and query head layouts disagree, or if they
|
||||
live on different devices.
|
||||
|
||||
"""
|
||||
sinks = self.sinks
|
||||
if sinks is None:
|
||||
|
||||
@@ -107,6 +107,7 @@ class HYV4HCPreLayer(nn.Module):
|
||||
Returns:
|
||||
A tuple of the pre-gated reduction ``[num_tokens, d]`` and the post
|
||||
gates ``[num_tokens, hc]`` consumed by `HYV4HCPostLayer`.
|
||||
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x)
|
||||
@@ -186,6 +187,7 @@ class HYV4HCPostLayer(nn.Module):
|
||||
|
||||
Returns:
|
||||
The updated residual channels ``[num_tokens, hc, d]``.
|
||||
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x, residual, post)
|
||||
@@ -272,6 +274,7 @@ class HYV4HCHeadLayer(nn.Module):
|
||||
|
||||
Returns:
|
||||
The merged hidden state ``[num_tokens, d]``.
|
||||
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x)
|
||||
@@ -367,6 +370,7 @@ class HYV4HCLayer(nn.Module):
|
||||
A tuple of the reduced hidden states ``[num_tokens, d]``, the post
|
||||
gates ``[num_tokens, hc]`` (``None`` when iHC is disabled) and the
|
||||
residual (the untouched input).
|
||||
|
||||
"""
|
||||
if not self.enable_ihc:
|
||||
return hidden_states, None, hidden_states
|
||||
|
||||
@@ -189,7 +189,7 @@ class HYV4DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
"""iHC forward: each sub-block reduces and re-scatters the channels."""
|
||||
"""IHC forward: each sub-block reduces and re-scatters the channels."""
|
||||
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
|
||||
hidden_states, post_gates, residual = self.hc_attn_layer.pre(hidden_states)
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
|
||||
@@ -106,6 +106,7 @@ def _resolve_fused_expert_param(
|
||||
|
||||
Returns:
|
||||
The matching draft parameter name, or None when there is none.
|
||||
|
||||
"""
|
||||
if not ckpt_suffix:
|
||||
return param_base if param_base in params_dict else None
|
||||
@@ -164,6 +165,7 @@ def _create_mtp_quant_config(
|
||||
Returns:
|
||||
The quantization config to use for the MTP layers, or None when the MTP
|
||||
layers are unquantized.
|
||||
|
||||
"""
|
||||
mtp_quant_algo = getattr(hf_config, "mtp_quant_algo", None)
|
||||
|
||||
@@ -241,6 +243,7 @@ def _remap_mtp_quant_exclusions(
|
||||
|
||||
Returns:
|
||||
A shallow copy with the translated exclusions, or the input unchanged.
|
||||
|
||||
"""
|
||||
if quant_config is None:
|
||||
return None
|
||||
@@ -671,6 +674,7 @@ class HYV4MTP(nn.Module):
|
||||
Returns:
|
||||
True when the weight was consumed (even if this rank holds none of
|
||||
the addressed experts).
|
||||
|
||||
"""
|
||||
base = name.split(".experts.")[0]
|
||||
for ckpt_proj, tag in (
|
||||
|
||||
@@ -37,6 +37,7 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
|
||||
logits_mup_width_multiplier: muP width divisor for the final logits;
|
||||
``None`` or ``0`` disables it.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -72,7 +72,8 @@ def _sconv_add_norm(
|
||||
|
||||
ROCm uses the portable collective path. The Lamport P2P implementation in
|
||||
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
|
||||
publication semantics which cannot be linked into a gfx950 HSACO."""
|
||||
publication semantics which cannot be linked into a gfx950 HSACO.
|
||||
"""
|
||||
norm_w = norm.weight if norm is not None else None
|
||||
eps = norm.variance_epsilon if norm is not None else 0.0
|
||||
|
||||
|
||||
@@ -192,7 +192,8 @@ def inkling_gate_select(
|
||||
|
||||
class InklingGate(nn.Module):
|
||||
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
|
||||
global scale (the served checkpoint's only configuration)."""
|
||||
global scale (the served checkpoint's only configuration).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -246,7 +247,8 @@ class InklingGate(nn.Module):
|
||||
self, gating_output: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
|
||||
are the routed top-k; the S trailing entries are the sink gammas."""
|
||||
are the routed top-k; the S trailing entries are the sink gammas.
|
||||
"""
|
||||
return inkling_gate_select(
|
||||
gating_output,
|
||||
self.n_total_experts,
|
||||
@@ -267,7 +269,8 @@ class InklingGate(nn.Module):
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the MoERunner layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled)."""
|
||||
expert parallelism is enabled).
|
||||
"""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
if not parallel_config.enable_expert_parallel:
|
||||
return 1
|
||||
|
||||
@@ -166,7 +166,8 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
|
||||
the target's tower embeddings scattered in unnormed (the backbone
|
||||
convention — MM embeds are merged after embed_norm)."""
|
||||
convention — MM embeds are merged after embed_norm).
|
||||
"""
|
||||
norm = self.backbone_embed_norm
|
||||
embeds = embed_rmsnorm(
|
||||
input_ids,
|
||||
@@ -195,7 +196,8 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
|
||||
launch: embedding row gather + the backbone embed_norm + the depth
|
||||
embed_norm chain on one side, hidden_norm on the other, written
|
||||
straight into the cat buffer."""
|
||||
straight into the cat buffer.
|
||||
"""
|
||||
hidden_w = layer.hidden_norm.weight
|
||||
embed_w = layer.embed_norm.weight
|
||||
eps = layer.hidden_norm.variance_epsilon
|
||||
@@ -327,7 +329,8 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
skipped entirely."""
|
||||
skipped entirely.
|
||||
"""
|
||||
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -238,7 +238,8 @@ def embed_rmsnorm(
|
||||
embedding-table gather.
|
||||
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
|
||||
first decoder layer's pre-attention norm) as a second output, still one
|
||||
launch. Bit-exact vs the unfused module sequence."""
|
||||
launch. Bit-exact vs the unfused module sequence.
|
||||
"""
|
||||
ids = input_ids.view(-1)
|
||||
(T,) = ids.shape
|
||||
n = embed_table.shape[1]
|
||||
@@ -332,7 +333,8 @@ def embed_dual_rmsnorm_cat(
|
||||
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
|
||||
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
|
||||
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
|
||||
unfused sequence). The concat copies collapse into direct writes."""
|
||||
unfused sequence). The concat copies collapse into direct writes.
|
||||
"""
|
||||
T, n = hidden.shape
|
||||
if embeds is not None:
|
||||
assert embeds.shape == hidden.shape
|
||||
|
||||
@@ -37,6 +37,7 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
|
||||
logits_mup_width_multiplier: muP width divisor for the final logits;
|
||||
``None`` or ``0`` disables it.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -75,7 +75,8 @@ def _sconv_add_norm(
|
||||
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
|
||||
|
||||
The Lamport path performs reduce-scatter + shard sconv + all-gather +
|
||||
residual add + norm. The NCCL path handles unsupported configurations."""
|
||||
residual add + norm. The NCCL path handles unsupported configurations.
|
||||
"""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
m = (
|
||||
attn_metadata.get(sconv.owner.prefix)
|
||||
|
||||
@@ -191,7 +191,8 @@ def inkling_gate_select(
|
||||
|
||||
class InklingGate(nn.Module):
|
||||
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
|
||||
global scale (the served checkpoint's only configuration)."""
|
||||
global scale (the served checkpoint's only configuration).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -243,7 +244,8 @@ class InklingGate(nn.Module):
|
||||
self, gating_output: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
|
||||
are the routed top-k; the S trailing entries are the sink gammas."""
|
||||
are the routed top-k; the S trailing entries are the sink gammas.
|
||||
"""
|
||||
return inkling_gate_select(
|
||||
gating_output,
|
||||
self.n_total_experts,
|
||||
@@ -264,7 +266,8 @@ class InklingGate(nn.Module):
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the FusedMoE layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled)."""
|
||||
expert parallelism is enabled).
|
||||
"""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
if not parallel_config.enable_expert_parallel:
|
||||
return 1
|
||||
|
||||
@@ -166,7 +166,8 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
|
||||
the target's tower embeddings scattered in unnormed (the backbone
|
||||
convention — MM embeds are merged after embed_norm)."""
|
||||
convention — MM embeds are merged after embed_norm).
|
||||
"""
|
||||
norm = self.backbone_embed_norm
|
||||
embeds = embed_rmsnorm(
|
||||
input_ids,
|
||||
@@ -195,7 +196,8 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
|
||||
launch: embedding row gather + the backbone embed_norm + the depth
|
||||
embed_norm chain on one side, hidden_norm on the other, written
|
||||
straight into the cat buffer."""
|
||||
straight into the cat buffer.
|
||||
"""
|
||||
hidden_w = layer.hidden_norm.weight
|
||||
embed_w = layer.embed_norm.weight
|
||||
eps = layer.hidden_norm.variance_epsilon
|
||||
@@ -327,7 +329,8 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
skipped entirely."""
|
||||
skipped entirely.
|
||||
"""
|
||||
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -60,7 +60,8 @@ def dmel_embed_sum_norm(
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
|
||||
[T, NB, D] intermediate)."""
|
||||
[T, NB, D] intermediate).
|
||||
"""
|
||||
T, nb = dmel_idx.shape
|
||||
D = weight.shape[1]
|
||||
assert weight.shape[0] % nb == 0
|
||||
@@ -151,7 +152,8 @@ def rmsnorm_gelu(
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
|
||||
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
|
||||
comes back as ``fold_timespace_to_depth(result, *fold)``."""
|
||||
comes back as ``fold_timespace_to_depth(result, *fold)``.
|
||||
"""
|
||||
D = x.shape[-1]
|
||||
flat = x.reshape(-1, D)
|
||||
assert flat.stride(1) == 1 and flat.stride(0) == D
|
||||
|
||||
@@ -238,7 +238,8 @@ def embed_rmsnorm(
|
||||
embedding-table gather.
|
||||
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
|
||||
first decoder layer's pre-attention norm) as a second output, still one
|
||||
launch. Bit-exact vs the unfused module sequence."""
|
||||
launch. Bit-exact vs the unfused module sequence.
|
||||
"""
|
||||
ids = input_ids.view(-1)
|
||||
(T,) = ids.shape
|
||||
n = embed_table.shape[1]
|
||||
@@ -332,7 +333,8 @@ def embed_dual_rmsnorm_cat(
|
||||
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
|
||||
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
|
||||
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
|
||||
unfused sequence). The concat copies collapse into direct writes."""
|
||||
unfused sequence). The concat copies collapse into direct writes.
|
||||
"""
|
||||
T, n = hidden.shape
|
||||
if embeds is not None:
|
||||
assert embeds.shape == hidden.shape
|
||||
|
||||
@@ -61,9 +61,7 @@ class ROCmLatentMoERunner(MoERunner):
|
||||
shared_output: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Tier 2: column-parallel up-projection folded into the final reduce.
|
||||
"""
|
||||
"""Tier 2: column-parallel up-projection folded into the final reduce."""
|
||||
if not self._logged_sharded_tail:
|
||||
self._logged_sharded_tail = True
|
||||
logger.info_once(
|
||||
|
||||
@@ -317,9 +317,7 @@ class KimiMoE(nn.Module):
|
||||
|
||||
|
||||
class KimiMLAAttention(nn.Module):
|
||||
"""
|
||||
Main reference: DeepseekV2 vllm Implementation
|
||||
"""
|
||||
"""Main reference: DeepseekV2 vllm Implementation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -213,9 +213,17 @@ def fused_kda_chunk(
|
||||
|
||||
Args:
|
||||
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
|
||||
w: chunk-local WY representation of the gated keys.
|
||||
u: chunk-local WY representation of the values.
|
||||
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
|
||||
aqk: intra-chunk query-key attention, ``[chunks, H, 64, 64]``.
|
||||
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
|
||||
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
|
||||
scale: scale applied to the query-key products.
|
||||
cu_seqlens: int32 cumulative sequence lengths.
|
||||
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
|
||||
output_final_state: whether to return the final recurrent state.
|
||||
chunk_offsets: int32 per-sequence first chunk index.
|
||||
checkpoint_state: destination for the mid-prefill state snapshots,
|
||||
fp32 ``[rows, H, 128, 128]``. Without
|
||||
``checkpoint_state_indices`` it is a staging buffer indexed by
|
||||
@@ -239,6 +247,7 @@ def fused_kda_chunk(
|
||||
has_initial_state: bool ``[N]``; a false entry starts that sequence
|
||||
from a zero state and its cache row is read only, never before the
|
||||
walk writes it.
|
||||
|
||||
"""
|
||||
if state_cache is not None:
|
||||
if initial_state is not None or output_final_state:
|
||||
|
||||
@@ -52,6 +52,22 @@ def chunk_kda_prefill(
|
||||
"""Run chunk KDA from raw gate and beta projections.
|
||||
|
||||
Args:
|
||||
q: query tensor, ``[1, T, H, K]``.
|
||||
k: key tensor, ``[1, T, H, K]``.
|
||||
v: value tensor, ``[1, T, H, V]``.
|
||||
raw_g: raw gate projection, before the activation.
|
||||
raw_beta: raw beta projection, before the activation.
|
||||
A_log: log of the per-head gate decay.
|
||||
g_bias: optional per-head gate bias.
|
||||
scale: scale applied to the query-key products. Defaults to
|
||||
``k.shape[-1] ** -0.5``.
|
||||
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
|
||||
output_final_state: whether to return the final recurrent state.
|
||||
lower_bound: optional floor applied to the gate.
|
||||
use_qk_l2norm_in_kernel: fold the q/k L2 norm into the kernel.
|
||||
cu_seqlens: int32 cumulative sequence lengths.
|
||||
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
|
||||
chunk_offsets: int32 per-sequence first chunk index.
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
@@ -71,6 +87,7 @@ def chunk_kda_prefill(
|
||||
|
||||
Returns:
|
||||
The output and, when requested, the final recurrent state.
|
||||
|
||||
"""
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
+6
-7
@@ -910,13 +910,12 @@ def fused_kda_gate(
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
|
||||
@@ -59,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
USE_SAFE_GATE: tl.constexpr,
|
||||
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
|
||||
|
||||
This kernel:
|
||||
|
||||
@@ -151,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
|
||||
chunk_size: int = 64,
|
||||
sub_chunk_size: int = 16,
|
||||
) -> None:
|
||||
"""
|
||||
Token-parallel implementation: each token gets its own thread block.
|
||||
"""Token-parallel implementation: each token gets its own thread block.
|
||||
Supports both fixed-length and variable-length sequences.
|
||||
Reduces wasted computation on padding.
|
||||
|
||||
@@ -164,10 +163,12 @@ def chunk_kda_fwd_intra_token_parallel(
|
||||
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
|
||||
beta: [B, T, HV]
|
||||
Aqk: [B, T, HV, BT] output tensor to write to
|
||||
cu_seqlens: cumulative sequence lengths for variable-length input
|
||||
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
|
||||
scale: attention scale
|
||||
chunk_size: BT (default 64)
|
||||
sub_chunk_size: BC (default 16)
|
||||
|
||||
"""
|
||||
B, T, H, K, HV = *q.shape, gk.shape[2]
|
||||
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
|
||||
|
||||
@@ -275,9 +275,7 @@ class LatentMoERunner(MoERunner):
|
||||
shared_output: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Tier 2: column-parallel up-projection folded into the final reduce.
|
||||
"""
|
||||
"""Tier 2: column-parallel up-projection folded into the final reduce."""
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
|
||||
|
||||
@@ -751,6 +751,7 @@ def run_kda_projection_overlap(
|
||||
|
||||
Returns:
|
||||
The Q/K/V/G projection, F_B output, and beta projection.
|
||||
|
||||
"""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
qkvg_weight = packed_weight[:_KDA_QKVG_SIZE]
|
||||
|
||||
@@ -745,7 +745,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused decode query-concat + latent cache insert, dispatched by cache
|
||||
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
|
||||
dtype (same policy as prefill: fp8 cache -> fp8 query).
|
||||
"""
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
cache = self.kv_cache
|
||||
if cache.dtype != torch.uint8:
|
||||
|
||||
@@ -339,6 +339,7 @@ class KimiRoutedOutputTransform(nn.Module):
|
||||
residual: Optional tensor of the up-projection's output shape to
|
||||
accumulate into. It is consumed in the GEMM's beta-add
|
||||
epilogue, so adding it costs no extra kernel.
|
||||
|
||||
"""
|
||||
if self.norm is not None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
@@ -796,6 +797,7 @@ class KimiMoE(nn.Module):
|
||||
``router_output`` holds the grouped top-k weights and ``topk_ids``
|
||||
the selected experts; otherwise ``router_output`` holds the raw gate
|
||||
logits and ``topk_ids`` is ``None``.
|
||||
|
||||
"""
|
||||
|
||||
def _router(
|
||||
|
||||
-3
@@ -59,7 +59,6 @@ def _mapping(
|
||||
|
||||
def validate_shape(*, tp_size: int, latent_dim: int, hidden_dim: int) -> None:
|
||||
"""Validate constraints imposed by the fused collective mapping."""
|
||||
|
||||
if tp_size <= 0 or latent_dim <= 0 or hidden_dim <= 0:
|
||||
raise ValueError("collective dimensions must be positive")
|
||||
if hidden_dim % tp_size:
|
||||
@@ -86,7 +85,6 @@ def _select_routed_schedule(
|
||||
max_m: int,
|
||||
) -> tuple[int, int]:
|
||||
"""Match upstream MNNVL's one-cluster-per-token occupancy policy."""
|
||||
|
||||
_, cluster_ctas, threads, _ = _mapping(tp_size, latent_dim, hidden_dim)
|
||||
sm_count = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
@@ -886,7 +884,6 @@ def compile_kernel(
|
||||
top_k: int = 0,
|
||||
) -> None:
|
||||
"""Compile the rank/M specialization without retaining caller tensors."""
|
||||
|
||||
key = _compile_key(
|
||||
rank,
|
||||
tp_size,
|
||||
|
||||
@@ -79,7 +79,6 @@ def validate_configuration(
|
||||
b_prime_stages: int,
|
||||
) -> None:
|
||||
"""Validate constraints imposed by this BF16 SM100 GEMM."""
|
||||
|
||||
if latent_dim <= 0 or shard_dim <= 0:
|
||||
raise ValueError("GEMM K and N must be positive")
|
||||
if latent_dim % 8 or shard_dim % 8:
|
||||
|
||||
@@ -52,7 +52,6 @@ def to_cute_dynamic_m(
|
||||
mode is symbolic, so changing M within an Op's capacity reuses the same
|
||||
compiled kernel.
|
||||
"""
|
||||
|
||||
return to_cute(tensor, assumed_align).mark_compact_shape_dynamic(
|
||||
mode=mode,
|
||||
stride_order=tensor.dim_order(),
|
||||
@@ -98,7 +97,6 @@ def load_global_u32x4(
|
||||
side-effecting prevents loop-invariant motion and common-subexpression
|
||||
elimination across polling iterations.
|
||||
"""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
|
||||
out = llvm.inline_asm(
|
||||
@@ -194,7 +192,6 @@ def finalize_top16_bf16(
|
||||
ip=None,
|
||||
):
|
||||
"""Finalize one Kimi K3 top-16 vector with wide metadata loads."""
|
||||
|
||||
addresses = [
|
||||
pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
|
||||
for pointer in (gemm2_vector, route_indices, route_weights)
|
||||
@@ -229,7 +226,6 @@ def store_global_u32x4(
|
||||
ip=None,
|
||||
) -> None:
|
||||
"""Store four packed words to an ordinary or NVLS multicast global VA."""
|
||||
|
||||
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
|
||||
opcode = "st.volatile.global.v4.u32" if volatile else "st.global.v4.u32"
|
||||
llvm.inline_asm(
|
||||
@@ -294,7 +290,6 @@ def store_global_u32(
|
||||
@dsl_user_op
|
||||
def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
|
||||
"""Publish eight BF16 values through an NVLS multicast mapping."""
|
||||
|
||||
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
@@ -312,7 +307,6 @@ def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
|
||||
@dsl_user_op
|
||||
def store_lamport_sentinel_128(pointer: cute.Pointer, *, loc=None, ip=None) -> None:
|
||||
"""Reset one Lamport fragment to four FP32 negative-zero bit patterns."""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
value = Uint32(NEG_ZERO_F32_BITS).ir_value(loc=loc, ip=ip)
|
||||
llvm.inline_asm(
|
||||
@@ -333,7 +327,6 @@ def red_async_release_gpu_add_u32(
|
||||
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
|
||||
) -> None:
|
||||
"""The exact SM100 arrival primitive used by upstream LamportFlags."""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
@@ -378,7 +371,6 @@ def map_shared_to_peer(
|
||||
ip=None,
|
||||
) -> Int32:
|
||||
"""Map a local shared-memory slot to the same slot in a peer CTA."""
|
||||
|
||||
smem_address = smem_ptr.toint(loc=loc, ip=ip).ir_value()
|
||||
return Int32(
|
||||
llvm.inline_asm(
|
||||
@@ -422,7 +414,6 @@ def store_shared_cluster_f32(
|
||||
@dsl_user_op
|
||||
def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
|
||||
"""Load two aligned FP32 DSM partials from local shared memory."""
|
||||
|
||||
address = Int32(pointer.toint(loc=loc, ip=ip))
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.f32()] * 2),
|
||||
@@ -443,7 +434,6 @@ def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
|
||||
@dsl_user_op
|
||||
def load_shared_f32x4(pointer: cute.Pointer, *, loc=None, ip=None):
|
||||
"""Load four aligned FP32 DSM partials from local shared memory."""
|
||||
|
||||
address = Int32(pointer.toint(loc=loc, ip=ip))
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.f32()] * 4),
|
||||
@@ -530,7 +520,6 @@ def sanitize_negative_zero_u32x2(packed):
|
||||
@cute.jit
|
||||
def sanitize_negative_zero(packed):
|
||||
"""Turn real BF16 -0 into +0 so it cannot equal the empty sentinel."""
|
||||
|
||||
result = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
word = packed[i]
|
||||
@@ -547,7 +536,6 @@ def sanitize_negative_zero(packed):
|
||||
@cute.jit
|
||||
def fragment_is_dirty(packed):
|
||||
"""Bit-exact upstream sentinel check: one comparison per 32-bit word."""
|
||||
|
||||
dirty = packed[0] == Uint32(NEG_ZERO_F32_BITS)
|
||||
for i in cutlass.range_constexpr(1, 4):
|
||||
dirty = dirty | (packed[i] == Uint32(NEG_ZERO_F32_BITS))
|
||||
@@ -564,7 +552,6 @@ def warp_sum_specialized(
|
||||
last_warp_mask: cutlass.Constexpr[int],
|
||||
) -> Float32:
|
||||
"""Warp sum supporting a compile-time partial final warp."""
|
||||
|
||||
if warp_idx == Int32(warps - 1) and cutlass.const_expr(last_warp_lanes < 32):
|
||||
for offset in cutlass.range_constexpr(16, 0, -1):
|
||||
# range_constexpr does not provide powers-of-two stepping.
|
||||
|
||||
+7
-10
@@ -539,9 +539,7 @@ def fused_kda_gate_chunk_cumsum(
|
||||
)
|
||||
B, T, H, D = raw_g.shape
|
||||
if raw_beta.shape != (B, T, H):
|
||||
raise ValueError(
|
||||
f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}"
|
||||
)
|
||||
raise ValueError(f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}")
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
|
||||
@@ -901,13 +899,12 @@ def fused_kda_gate(
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
|
||||
+200
-98
@@ -23,18 +23,20 @@ from .chunk_intra_token_parallel import chunk_kda_fwd_intra_token_parallel
|
||||
################################################################################
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({'BK': BK}, num_warps=num_warps)
|
||||
triton.Config({"BK": BK}, num_warps=num_warps)
|
||||
for BK in [32, 64]
|
||||
for num_warps in [1, 2, 4]
|
||||
],
|
||||
key=["H", "HV", "K", "BC"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['T'])
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
q,
|
||||
k,
|
||||
@@ -57,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
USE_SAFE_GATE: tl.constexpr,
|
||||
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
|
||||
|
||||
This kernel:
|
||||
@@ -74,8 +75,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
@@ -122,21 +129,31 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
|
||||
p_k0 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g0 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k0 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g0 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
if i_tc1 < T:
|
||||
p_q1 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k1 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g1 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
p_q1 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k1 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g1 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn1 = tl.load(g + i_tc1 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
b_gqn = tl.where(m_tc1[:, None], exp2(b_g1 - b_gn1[None, :]), 0)
|
||||
# [BK, BC]
|
||||
@@ -146,15 +163,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
|
||||
|
||||
if i_tc2 < T:
|
||||
p_q2 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k2 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g2 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
p_q2 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k2 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g2 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn2 = tl.load(g + i_tc2 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
# [BC, BK]
|
||||
b_gqn2 = tl.where(m_tc2[:, None], exp2(b_g2 - b_gn2[None, :]), 0)
|
||||
b_qg2 = b_q2 * b_gqn2
|
||||
@@ -170,15 +195,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Akk21 += tl.dot(b_kg2, b_kgt)
|
||||
|
||||
if i_tc3 < T:
|
||||
p_q3 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k3 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g3 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
p_q3 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k3 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g3 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn3 = tl.load(g + i_tc3 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
# [BC, BK]
|
||||
b_gqn3 = tl.where(m_tc3[:, None], exp2(b_g3 - b_gn3[None, :]), 0)
|
||||
b_qg3 = b_q3 * b_gqn3
|
||||
@@ -203,40 +236,78 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
# save off-diagonal Aqk blocks and prepare Akk
|
||||
################################################################################
|
||||
if i_tc1 < T:
|
||||
p_Aqk10 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_Aqk10 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b1 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,))
|
||||
p_b1 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
|
||||
)
|
||||
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk10 = b_Akk10 * b_b1[:, None]
|
||||
if i_tc2 < T:
|
||||
p_Aqk20 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Aqk21 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_Aqk20 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk21 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b2 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,))
|
||||
p_b2 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
|
||||
)
|
||||
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk20 = b_Akk20 * b_b2[:, None]
|
||||
b_Akk21 = b_Akk21 * b_b2[:, None]
|
||||
if i_tc3 < T:
|
||||
p_Aqk30 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
p_Aqk31 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
|
||||
p_Aqk32 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_Aqk30 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk31 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk32 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b3 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,))
|
||||
p_b3 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
|
||||
)
|
||||
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk30 = b_Akk30 * b_b3[:, None]
|
||||
b_Akk31 = b_Akk31 * b_b3[:, None]
|
||||
b_Akk32 = b_Akk32 * b_b3[:, None]
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc0, 0), (BC, BC), (1, 0))
|
||||
p_Akk11 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
p_Akk22 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Akk33 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
|
||||
@@ -256,25 +327,25 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Ai33 = -tl.where(m_A, b_Ai33, 0)
|
||||
|
||||
for i in range(2, min(BC, T - i_tc0)):
|
||||
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a00 = tl.where(o_i < i, b_a00, 0.)
|
||||
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a00 = tl.where(o_i < i, b_a00, 0.0)
|
||||
b_a00 += tl.sum(b_a00[:, None] * b_Ai00, 0)
|
||||
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
|
||||
for i in range(BC + 2, min(2*BC, T - i_tc0)):
|
||||
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a11 = tl.where(o_i < i - BC, b_a11, 0.)
|
||||
for i in range(BC + 2, min(2 * BC, T - i_tc0)):
|
||||
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a11 = tl.where(o_i < i - BC, b_a11, 0.0)
|
||||
b_a11 += tl.sum(b_a11[:, None] * b_Ai11, 0)
|
||||
b_Ai11 = tl.where((o_i == i - BC)[:, None], b_a11, b_Ai11)
|
||||
for i in range(2*BC + 2, min(3*BC, T - i_tc0)):
|
||||
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a22 = tl.where(o_i < i - 2*BC, b_a22, 0.)
|
||||
for i in range(2 * BC + 2, min(3 * BC, T - i_tc0)):
|
||||
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a22 = tl.where(o_i < i - 2 * BC, b_a22, 0.0)
|
||||
b_a22 += tl.sum(b_a22[:, None] * b_Ai22, 0)
|
||||
b_Ai22 = tl.where((o_i == i - 2*BC)[:, None], b_a22, b_Ai22)
|
||||
for i in range(3*BC + 2, min(4*BC, T - i_tc0)):
|
||||
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a33 = tl.where(o_i < i - 3*BC, b_a33, 0.)
|
||||
b_Ai22 = tl.where((o_i == i - 2 * BC)[:, None], b_a22, b_Ai22)
|
||||
for i in range(3 * BC + 2, min(4 * BC, T - i_tc0)):
|
||||
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a33 = tl.where(o_i < i - 3 * BC, b_a33, 0.0)
|
||||
b_a33 += tl.sum(b_a33[:, None] * b_Ai33, 0)
|
||||
b_Ai33 = tl.where((o_i == i - 3*BC)[:, None], b_a33, b_Ai33)
|
||||
b_Ai33 = tl.where((o_i == i - 3 * BC)[:, None], b_a33, b_Ai33)
|
||||
|
||||
b_Ai00 += m_I
|
||||
b_Ai11 += m_I
|
||||
@@ -289,53 +360,73 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Ai10 = -tl.dot(
|
||||
tl.dot(b_Ai11, b_Akk10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai00,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai21 = -tl.dot(
|
||||
tl.dot(b_Ai22, b_Akk21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai11,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai32 = -tl.dot(
|
||||
tl.dot(b_Ai33, b_Akk32, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai22,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
|
||||
b_Ai20 = -tl.dot(
|
||||
b_Ai22,
|
||||
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai31 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai30 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
|
||||
################################################################################
|
||||
# store full Akk_inv to Akk
|
||||
################################################################################
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
|
||||
p_Akk10 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
p_Akk11 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
|
||||
p_Akk20 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Akk21 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
|
||||
p_Akk22 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 2*BC), (BC, BC), (1, 0))
|
||||
p_Akk30 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
p_Akk31 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
|
||||
p_Akk32 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
|
||||
p_Akk33 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 3*BC), (BC, BC), (1, 0))
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk10 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk20 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk21 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk30 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk31 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk32 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
|
||||
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
@@ -349,18 +440,20 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=['BK', 'NC', 'BT', 'HV'],
|
||||
key=["BK", "NC", "BT", "HV"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['B', 'T'])
|
||||
@triton.jit(do_not_specialize=["B", "T"])
|
||||
def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
q,
|
||||
k,
|
||||
@@ -386,8 +479,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
@@ -406,9 +505,9 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
Aqk = Aqk + (bos * HV + i_hv) * BT
|
||||
Akk = Akk + (bos * HV + i_hv) * BC
|
||||
|
||||
p_q = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_k = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
|
||||
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
|
||||
|
||||
@@ -418,10 +517,12 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
|
||||
if USE_GATHER:
|
||||
b_gn = gather(b_g, tl.full([1, BK], min(BC//2, T - i_ti - 1), dtype=tl.int16), axis=0)
|
||||
b_gn = gather(
|
||||
b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0
|
||||
)
|
||||
else:
|
||||
# caculate offset
|
||||
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV*K + tl.arange(0, BK)
|
||||
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + tl.arange(0, BK)
|
||||
b_gn = tl.load(p_gn, mask=tl.arange(0, BK) < K, other=0.0)
|
||||
b_gn = b_gn[None, :]
|
||||
|
||||
@@ -429,8 +530,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
# less than 85 to avoid overflow in exp2
|
||||
b_gm = (b_g - b_gn).to(tl.float32)
|
||||
|
||||
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.)
|
||||
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.)
|
||||
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.0)
|
||||
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.0)
|
||||
|
||||
b_kgt = tl.trans(b_k * b_gk)
|
||||
|
||||
@@ -445,8 +546,10 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
|
||||
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
|
||||
|
||||
p_Aqk = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0))
|
||||
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV*BC, 1), (i_ti, 0), (BC, BC), (1, 0))
|
||||
p_Aqk = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
@@ -458,8 +561,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
|
||||
b_Ai = -b_Akk
|
||||
for i in range(2, min(BC, T - i_ti)):
|
||||
b_a = -tl.load(Akk + (i_ti + i) * HV*BC + o_i)
|
||||
b_a = tl.where(o_i < i, b_a, 0.)
|
||||
b_a = -tl.load(Akk + (i_ti + i) * HV * BC + o_i)
|
||||
b_a = tl.where(o_i < i, b_a, 0.0)
|
||||
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
|
||||
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
|
||||
b_Ai += m_I
|
||||
@@ -531,8 +634,7 @@ def chunk_kda_fwd_intra(
|
||||
# Step 2: Fused inter + solve_tril (works for both fixed-len and varlen)
|
||||
solve_tril_dot_precision = (
|
||||
"tf32"
|
||||
if current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(80)
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(80)
|
||||
else "ieee"
|
||||
)
|
||||
grid = (NT, B * HV)
|
||||
|
||||
+38
-18
@@ -16,18 +16,20 @@ from vllm.third_party.flash_linear_attention.ops.op import exp2
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({'BH': BH}, num_warps=num_warps)
|
||||
triton.Config({"BH": BH}, num_warps=num_warps)
|
||||
for BH in [1, 2, 4, 8]
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
],
|
||||
key=["K", "H", "HV"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['T', 'N'])
|
||||
@triton.jit(do_not_specialize=["T", "N"])
|
||||
def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
q,
|
||||
k,
|
||||
@@ -65,7 +67,10 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
left = mid + 1
|
||||
i_n = left
|
||||
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
i_t = i_tg - bos
|
||||
else:
|
||||
@@ -82,11 +87,11 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
|
||||
G: tl.constexpr = HV // H
|
||||
|
||||
q += bos * H*K
|
||||
k += bos * H*K
|
||||
g += bos * HV*K
|
||||
Aqk += bos * HV*BT
|
||||
Akk += bos * HV*BC
|
||||
q += bos * H * K
|
||||
k += bos * H * K
|
||||
g += bos * HV * K
|
||||
Aqk += bos * HV * BT
|
||||
Akk += bos * HV * BC
|
||||
beta += bos * HV
|
||||
|
||||
BK: tl.constexpr = triton.next_power_of_2(K)
|
||||
@@ -103,7 +108,9 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
|
||||
# g: [B, T, HV, K], beta: [B, T, HV]
|
||||
p_g = tl.make_block_ptr(g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(
|
||||
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
@@ -111,15 +118,25 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
|
||||
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
|
||||
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
p_gj = tl.make_block_ptr(g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
|
||||
p_gj = tl.make_block_ptr(
|
||||
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
|
||||
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
|
||||
b_Akk = tl.sum(b_k * b_kgj, axis=1) * tl.where(j < i_t, 1.0, 0.0)
|
||||
|
||||
tl.store(Aqk + i_t * HV * BT + o_hv * BT + j % BT, b_Aqk.to(Aqk.dtype.element_ty), mask=m_hv)
|
||||
tl.store(Akk + i_t * HV * BC + o_hv * BC + j - i_ts, b_Akk.to(Akk.dtype.element_ty), mask=m_hv)
|
||||
tl.store(
|
||||
Aqk + i_t * HV * BT + o_hv * BT + j % BT,
|
||||
b_Aqk.to(Aqk.dtype.element_ty),
|
||||
mask=m_hv,
|
||||
)
|
||||
tl.store(
|
||||
Akk + i_t * HV * BC + o_hv * BC + j - i_ts,
|
||||
b_Akk.to(Akk.dtype.element_ty),
|
||||
mask=m_hv,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra_token_parallel(
|
||||
@@ -134,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
|
||||
chunk_size: int = 64,
|
||||
sub_chunk_size: int = 16,
|
||||
) -> None:
|
||||
"""
|
||||
Token-parallel implementation: each token gets its own thread block.
|
||||
"""Token-parallel implementation: each token gets its own thread block.
|
||||
Supports both fixed-length and variable-length sequences.
|
||||
Reduces wasted computation on padding.
|
||||
|
||||
@@ -147,17 +163,21 @@ def chunk_kda_fwd_intra_token_parallel(
|
||||
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
|
||||
beta: [B, T, HV]
|
||||
Aqk: [B, T, HV, BT] output tensor to write to
|
||||
cu_seqlens: cumulative sequence lengths for variable-length input
|
||||
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
|
||||
scale: attention scale
|
||||
chunk_size: BT (default 64)
|
||||
sub_chunk_size: BC (default 16)
|
||||
|
||||
"""
|
||||
B, T, H, K, HV = *q.shape, gk.shape[2]
|
||||
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
|
||||
BT = chunk_size
|
||||
BC = sub_chunk_size
|
||||
|
||||
def grid(meta): return (B * T, triton.cdiv(HV, meta['BH']))
|
||||
def grid(meta):
|
||||
return (B * T, triton.cdiv(HV, meta["BH"]))
|
||||
|
||||
chunk_kda_fwd_kernel_intra_token_parallel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
|
||||
@@ -483,9 +483,7 @@ def fused_recurrent_kda(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=["stride_beta_token", "stride_state_indices"]
|
||||
)
|
||||
@triton.jit(do_not_specialize=["stride_beta_token", "stride_state_indices"])
|
||||
def fused_recurrent_kda_packed_decode_kernel(
|
||||
mixed_qkv,
|
||||
raw_g,
|
||||
|
||||
@@ -334,7 +334,8 @@ def _aiter_moe_fused_shared_experts_enabled(
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
|
||||
expert."""
|
||||
expert.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -173,7 +173,8 @@ class MiniMaxM3IndexerPrefillMetadata:
|
||||
@dataclass
|
||||
class MiniMaxM3IndexerDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
per-request query length (1, or 1 + num_speculative_tokens).
|
||||
"""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
@@ -206,7 +207,8 @@ class MiniMaxM3IndexerMetadataBuilder(
|
||||
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
|
||||
):
|
||||
"""Abstract base: shared setup only. The Triton and MSA builders are
|
||||
parallel subclasses that each own their full ``build`` (no shared code)."""
|
||||
parallel subclasses that each own their full ``build`` (no shared code).
|
||||
"""
|
||||
|
||||
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
@@ -203,7 +203,8 @@ class MiniMaxM3SparsePrefillMetadata:
|
||||
@dataclass
|
||||
class MiniMaxM3SparseDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
per-request query length (1, or 1 + num_speculative_tokens).
|
||||
"""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
|
||||
@@ -85,7 +85,8 @@ class MiniMaxM3IndexerMSAPrefillMetadata:
|
||||
class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
|
||||
"""Decode reuses the inherited base ``decode`` field (the Triton decode
|
||||
metadata); ``prefill_msa`` carries the fmha score plan for the prefill side
|
||||
(the base ``prefill`` field is unused on this path)."""
|
||||
(the base ``prefill`` field is unused on this path).
|
||||
"""
|
||||
|
||||
prefill_msa: MiniMaxM3IndexerMSAPrefillMetadata | None = None
|
||||
# Per-forward view (``[:num_tokens]``) of the builder's persistent unified
|
||||
@@ -111,7 +112,8 @@ class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
|
||||
|
||||
class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
"""Decode metadata is the cudagraph-safe Triton decode metadata; the prefill
|
||||
fmha plan is built eagerly (prefill batches are not captured)."""
|
||||
fmha plan is built eagerly (prefill batches are not captured).
|
||||
"""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
|
||||
@@ -190,7 +190,8 @@ class MiniMaxM3MLP(nn.Module):
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
|
||||
expert."""
|
||||
expert.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -782,7 +783,8 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
@property
|
||||
def ffn_all_reduce_deferred(self) -> bool:
|
||||
"""This layer's FFN output is left un-reduced; the caller fuses the
|
||||
all-reduce into the next RMSNorm."""
|
||||
all-reduce into the next RMSNorm.
|
||||
"""
|
||||
if self.is_moe_layer:
|
||||
return self.block_sparse_moe.experts.moe_config.skip_final_all_reduce
|
||||
return not self.mlp.down_proj.reduce_results
|
||||
|
||||
@@ -33,7 +33,6 @@ def apply_qsa_rope(
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
|
||||
|
||||
num_tokens, _, head_dim = tensor.shape
|
||||
rotary_dim = rotary_emb.rotary_dim
|
||||
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
|
||||
@@ -67,7 +66,6 @@ def apply_qsa_rmsnorm(
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Use vLLM's portable RMSNorm implementation on ROCm."""
|
||||
|
||||
return cast(torch.Tensor, norm(tensor))
|
||||
|
||||
|
||||
@@ -153,7 +151,6 @@ class QSAIndexer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
|
||||
|
||||
qk, _ = self.index_qk_proj(hidden_states)
|
||||
q_raw, token_k = qk.split(
|
||||
(
|
||||
@@ -176,7 +173,6 @@ class QSAIndexer(nn.Module):
|
||||
first_rope_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize pooled K and apply the first token's exact group position."""
|
||||
|
||||
keys = compressed_keys.reshape(-1, self.index_head_dim)
|
||||
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
|
||||
-1, 1, self.index_head_dim
|
||||
@@ -284,7 +280,6 @@ class QSAIndexer(nn.Module):
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``."""
|
||||
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
|
||||
@@ -11,5 +11,4 @@ def enable_qwen4_exp_low_latency_gemm(
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Keep the standard vLLM linear methods on AMD ROCm."""
|
||||
|
||||
del module, dtype
|
||||
|
||||
@@ -87,7 +87,6 @@ def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4."""
|
||||
|
||||
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
|
||||
return None
|
||||
return quant_config
|
||||
@@ -103,7 +102,6 @@ def _remap_qsa_cache_scale_name(
|
||||
that cache directly, so only QSA layers need the final path component
|
||||
moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers.
|
||||
"""
|
||||
|
||||
scale_suffixes = {
|
||||
"k_proj.k_scale": "_k_scale",
|
||||
"k_proj.output_scale": "_k_scale",
|
||||
|
||||
@@ -91,7 +91,6 @@ def _remap_quantized_layers(
|
||||
|
||||
def _remap_mtp_weight_name(name: str) -> str | None:
|
||||
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
|
||||
|
||||
for checkpoint_prefix in (
|
||||
"model.language_model.",
|
||||
"language_model.",
|
||||
@@ -260,7 +259,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
|
||||
def _iter_qsa_attentions(self):
|
||||
"""Yield MTP attention modules that own a QSA indexer."""
|
||||
|
||||
for layer in self.layers:
|
||||
attention = getattr(layer, "self_attn", None)
|
||||
if (
|
||||
@@ -271,13 +269,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
|
||||
|
||||
for attention in self._iter_qsa_attentions():
|
||||
attention.indexer.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
|
||||
"""Keep each request's target-aligned step-0 sparse-index row."""
|
||||
|
||||
num_rows = row_indices.numel()
|
||||
for attention in self._iter_qsa_attentions():
|
||||
buffer = attention.topk_indices_buffer
|
||||
|
||||
@@ -600,7 +600,6 @@ def qsa_mqa_paged(
|
||||
score_scale: float | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute QSA scores directly from a paged compressed-key cache."""
|
||||
|
||||
_validate_mqa(q)
|
||||
if not q.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("paged QSA scoring requires a GPU and Triton")
|
||||
@@ -678,7 +677,6 @@ def expand_qsa_block_indices_cuda(
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Expand compressed blocks and compact the causal tail of the open group."""
|
||||
|
||||
if not block_indices.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("QSA index expansion requires a GPU and Triton")
|
||||
if token_topk % compress_ratio:
|
||||
@@ -738,7 +736,6 @@ def qsa_select_paged_tokens(
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Score, select, and expand QSA indices without host synchronization."""
|
||||
|
||||
rows = q.shape[0]
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
if out is None:
|
||||
@@ -829,7 +826,6 @@ def qsa_sparse_paged_attention(
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run sparse GQA directly over paged BF16 K/V caches."""
|
||||
|
||||
if not q.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("paged QSA sparse attention requires a GPU and Triton")
|
||||
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
|
||||
@@ -971,7 +967,6 @@ def qsa_store_cache_rows(
|
||||
rows: torch.Tensor,
|
||||
) -> None:
|
||||
"""Store fixed-width rows in a QSA cache without boolean indexing."""
|
||||
|
||||
if not cache.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("QSA cache stores require a GPU and Triton")
|
||||
if cache.ndim != 4 or cache.shape[2] != 1:
|
||||
@@ -1017,7 +1012,6 @@ def qsa_compress_groups_with_ratio(
|
||||
rope_cache: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pool completed groups from the compressor-state ring and raw token rows."""
|
||||
|
||||
if not raw_keys.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("QSA compression requires a GPU and Triton")
|
||||
rows = token_to_req.numel()
|
||||
|
||||
@@ -357,7 +357,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
|
||||
persistent_buffers = {
|
||||
"layer_multipliers": self.layer_multipliers,
|
||||
"ngram_heads_offsets": self.ngram_heads_offsets,
|
||||
|
||||
@@ -430,7 +430,6 @@ def qwen4_exp_qsa_with_output(
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the complete QSA state/update/attend transaction."""
|
||||
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
|
||||
@@ -28,7 +28,6 @@ def compute_ple_shard_overlap(
|
||||
tp_end: int,
|
||||
) -> PLEShardOverlap | None:
|
||||
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
|
||||
|
||||
if checkpoint_start < 0 or checkpoint_rows < 0:
|
||||
raise ValueError("checkpoint shard bounds must be non-negative")
|
||||
if tp_start < 0 or tp_end < tp_start:
|
||||
@@ -54,7 +53,6 @@ def copy_ple_embedding_shard_(
|
||||
tp_end: int,
|
||||
) -> int:
|
||||
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
|
||||
|
||||
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
|
||||
raise ValueError("destination and loaded weight must have matching ranks")
|
||||
if destination.shape[1:] != loaded_weight.shape[1:]:
|
||||
|
||||
@@ -47,7 +47,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
|
||||
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
|
||||
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
|
||||
|
||||
if positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0).expand(3, -1)
|
||||
elif positions.ndim != 2 or positions.shape[0] not in (1, 3):
|
||||
@@ -115,7 +114,6 @@ def circular_qsa_slot_mapping(
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Map each request to its fixed physical block as a circular token ring."""
|
||||
|
||||
if compressor_state_size <= 0:
|
||||
raise ValueError("QSA circular buffer size must be positive")
|
||||
if block_table.ndim != 2:
|
||||
@@ -167,7 +165,6 @@ def compressed_qsa_slot_mapping(
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build boundary-only slots for an ``MLAAttentionSpec`` QSA cache."""
|
||||
|
||||
if storage_block_size <= 0 or compress_ratio <= 0:
|
||||
raise ValueError("QSA block size and compression ratio must be positive")
|
||||
compressed_positions = torch.div(
|
||||
|
||||
@@ -32,7 +32,6 @@ def apply_qsa_rope(
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
|
||||
|
||||
num_tokens, _, head_dim = tensor.shape
|
||||
rotary_dim = rotary_emb.rotary_dim
|
||||
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
|
||||
@@ -245,7 +244,6 @@ class QSAIndexer(nn.Module):
|
||||
valid-entry count (the attention kernel's loop bound, never a token
|
||||
index).
|
||||
"""
|
||||
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
|
||||
@@ -86,7 +86,6 @@ def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4."""
|
||||
|
||||
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
|
||||
return None
|
||||
return quant_config
|
||||
@@ -102,7 +101,6 @@ def _remap_qsa_cache_scale_name(
|
||||
that cache directly, so only QSA layers need the final path component
|
||||
moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers.
|
||||
"""
|
||||
|
||||
scale_suffixes = {
|
||||
"k_proj.k_scale": "_k_scale",
|
||||
"k_proj.output_scale": "_k_scale",
|
||||
|
||||
@@ -90,7 +90,6 @@ def _remap_quantized_layers(
|
||||
|
||||
def _remap_mtp_weight_name(name: str) -> str | None:
|
||||
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
|
||||
|
||||
for checkpoint_prefix in (
|
||||
"model.language_model.",
|
||||
"language_model.",
|
||||
@@ -250,7 +249,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
|
||||
def _iter_qsa_attentions(self):
|
||||
"""Yield MTP attention modules that own a QSA indexer."""
|
||||
|
||||
for layer in self.layers:
|
||||
attention = getattr(layer, "self_attn", None)
|
||||
if (
|
||||
@@ -261,13 +259,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
|
||||
|
||||
for attention in self._iter_qsa_attentions():
|
||||
attention.indexer.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
|
||||
"""Keep each request's target-aligned step-0 sparse-index row."""
|
||||
|
||||
num_rows = row_indices.numel()
|
||||
for attention in self._iter_qsa_attentions():
|
||||
buffer = attention.topk_indices_buffer
|
||||
|
||||
@@ -869,7 +869,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
|
||||
persistent_buffers = {
|
||||
"layer_multipliers": self.layer_multipliers,
|
||||
"ngram_heads_offsets": self.ngram_heads_offsets,
|
||||
|
||||
@@ -727,7 +727,6 @@ def warmup_qsa_sparse_paged_attention(
|
||||
selection_width: int,
|
||||
) -> tuple[tuple[int, int, int], ...]:
|
||||
"""Compile every production-reachable split-K/merge specialization."""
|
||||
|
||||
head_dim = kv_cache.shape[-1] // 2
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
|
||||
# An fp8 cache is allocated as uint8 and viewed as e4m3 at attention time.
|
||||
@@ -870,7 +869,6 @@ def qsa_store_cache_rows(
|
||||
rows: torch.Tensor,
|
||||
) -> None:
|
||||
"""Store fixed-width rows in a QSA cache without boolean indexing."""
|
||||
|
||||
if not cache.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("QSA CUDA cache stores require Triton")
|
||||
if cache.ndim != 4 or cache.shape[2] != 1:
|
||||
@@ -916,7 +914,6 @@ def qsa_compress_groups_with_ratio(
|
||||
rope_cache: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pool completed groups from the compressor-state ring and raw token rows."""
|
||||
|
||||
if not raw_keys.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("QSA CUDA compression requires Triton")
|
||||
rows = token_to_req.numel()
|
||||
|
||||
@@ -317,7 +317,6 @@ def warmup_qsa_mqa_paged_decode(
|
||||
max_num_batched_tokens: int,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
"""Compile every reachable decode specialization without launching it."""
|
||||
|
||||
page_size = k_cache.shape[1]
|
||||
page_table_width = page_table.shape[1]
|
||||
columns = page_table_width * page_size
|
||||
@@ -444,7 +443,6 @@ def expand_qsa_block_indices(
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Expand compressed blocks and compact the causal tail of the open group."""
|
||||
|
||||
assert token_topk % compress_ratio == 0
|
||||
block_topk = token_topk // compress_ratio
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
@@ -524,8 +522,8 @@ def qsa_select_paged_decode(
|
||||
compress_ratio: Number of logical tokens represented by a cache row.
|
||||
decode_query_len: Number of query tokens per request.
|
||||
block_indices: Compressed-index output buffer.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert token_topk % compress_ratio == 0
|
||||
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
|
||||
assert decode_query_len > 0 and q.shape[0] % decode_query_len == 0
|
||||
@@ -599,8 +597,8 @@ def qsa_select_paged_prefill(
|
||||
max_query_len: Maximum number of query tokens in one request.
|
||||
block_indices: Compressed-index output buffer.
|
||||
max_seq_len: Longest context length in the batch this step.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert token_topk % compress_ratio == 0
|
||||
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
|
||||
assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
|
||||
|
||||
@@ -146,7 +146,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize PLE lookup output."""
|
||||
|
||||
return self.ple_embedding.ngram_embedding.dequantize(
|
||||
embeddings,
|
||||
output_dtype,
|
||||
|
||||
Reference in New Issue
Block a user