Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
+2 -1
View File
@@ -68,7 +68,8 @@ class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKe
weight_ptr = kv_weight_ptr
row_out = kv_out_ptr + token_idx * kv_out_stride
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
# RMSNorm in fp32 throughout — matches
# csrc/layernorm_kernels.cu's
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
# keep x, rrms, and w all in fp32 and perform a single cast at store.
block = tl.arange(0, BLOCK_SIZE)
+33 -2
View File
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
@@ -179,6 +182,8 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
else:
attn_metadata = attn_metadata_raw
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
@@ -213,6 +218,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
mla_kv_cache = self.kv_cache
mla_k_scale = self._k_scale
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
kv_c_out = torch.empty_like(kv_c) if use_mha else None
k_pe_out = torch.empty_like(k_pe) if use_mha else None
q_c = fused_norm_rope(
positions,
q_c,
@@ -237,9 +246,18 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
kv_c_out=kv_c_out,
k_pe_out=k_pe_out,
)
ql_nope, q_pe = self._compute_ql_nope(q_c)
if use_mha:
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split(
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
)
ql_nope = q_nope
else:
ql_nope, q_pe = self._compute_ql_nope(q_c)
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
@@ -260,7 +278,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
indexer_n_head_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
quantize_mqa=self._fp8_kv,
quantize_mqa=self._fp8_kv and not use_mha,
)
if self.indexer is not None and not self.skip_topk:
@@ -270,6 +288,19 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
output.zero_()
return
if use_mha:
assert kv_c_out is not None and k_pe_out is not None
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
self.forward_impl(
mha_q,
kv_c_out,
k_pe_out.unsqueeze(1),
self.kv_cache,
attn_metadata,
output,
)
return
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
kv_cache = self.kv_cache
if self._fp8_kv_needs_view:
+13 -1
View File
@@ -6,7 +6,9 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.ops.common import pack_ue8m0_x4
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
# has_indexer=False path so the indexer args don't allocate every call.
@@ -87,6 +89,7 @@ def _fp8_quant_and_cache_write(
SHUFFLE: tl.constexpr,
BLOCK_TILE: tl.constexpr,
HEAD_TILE: tl.constexpr,
MX_SCALE: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FNUZ: tl.constexpr,
):
@@ -111,6 +114,8 @@ def _fp8_quant_and_cache_write(
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
if MX_SCALE:
scale = pack_ue8m0_x4(scale)
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
@@ -167,6 +172,7 @@ def _fused_norm_rope_kernel(
INDEXER_CACHE_SHUFFLE: tl.constexpr,
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
INDEXER_CACHE_MX_SCALE: tl.constexpr,
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
mla_cache_ptr,
mla_cache_block_stride,
@@ -501,6 +507,7 @@ def _fused_norm_rope_kernel(
INDEXER_CACHE_SHUFFLE,
INDEXER_CACHE_BLOCK_TILE,
INDEXER_CACHE_HEAD_TILE,
INDEXER_CACHE_MX_SCALE,
FP8_MAX,
USE_FNUZ,
)
@@ -571,7 +578,10 @@ def fused_norm_rope(
assert slot_mapping is not None
if indexer_slot_mapping is None:
indexer_slot_mapping = slot_mapping
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
idx_cache_mx_scale = mqa_logits_use_mx_scales()
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(
torch.int32 if idx_cache_mx_scale else torch.float32
)
idx_cache_block_size = indexer_k_cache.shape[1]
idx_cache_stride = indexer_k_cache.shape[2]
# The caller's cache reports whether its reader expects the shuffled
@@ -591,6 +601,7 @@ def fused_norm_rope(
)
else:
idx_cache_scale_view = None
idx_cache_mx_scale = False
idx_cache_block_size = 1
idx_cache_stride = 0
idx_cache_shuffle = False
@@ -701,6 +712,7 @@ def fused_norm_rope(
idx_cache_shuffle,
_INDEXER_CACHE_BLOCK_TILE,
idx_cache_head_tile,
idx_cache_mx_scale,
# MLA KV cache (uses same slot_mapping)
mla_kv_cache,
mla_block_stride,
-15
View File
@@ -26,10 +26,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
@@ -413,12 +409,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -474,11 +464,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
+163 -16
View File
@@ -10,8 +10,50 @@ import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
scaled_dequantize,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.utils.math_utils import next_power_of_2
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
@triton.jit
def _append_shared_expert_kernel(
topk_ids_ptr,
topk_weights_ptr,
out_ids_ptr,
out_weights_ptr,
num_tokens,
shared_expert_id,
TOPK: tl.constexpr,
ROUTED_PER_RANK: tl.constexpr,
BLOCK_M: tl.constexpr,
):
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
mask = rows < num_tokens
for k in tl.static_range(TOPK):
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
tl.store(
out_ids_ptr + rows * (TOPK + 1) + TOPK,
tl.zeros_like(rows) + shared_expert_id,
mask=mask,
)
tl.store(
out_weights_ptr + rows * (TOPK + 1) + TOPK,
tl.full([BLOCK_M], 1.0, tl.float32),
mask=mask,
)
class DeepseekV4AiterMegaMoEExperts(nn.Module):
@@ -28,16 +70,25 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
hidden_size: int,
intermediate_size: int,
swiglu_limit: float | None,
fuse_shared_expert: bool = False,
prefix: str = "",
):
super().__init__()
ep_group = get_ep_group()
self.ep_rank = ep_group.rank_in_group
self.ep_size = ep_group.world_size
self.num_experts = num_experts
self.num_local_experts = num_experts // self.ep_size
self.experts_start_idx = self.ep_rank * self.num_local_experts
self.top_k = top_k
self.num_routed_local_experts = num_experts // self.ep_size
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
# A fused shared expert is one extra local expert per rank and one
# extra route per token, to the token's own rank.
self.fuse_shared_expert = fuse_shared_expert
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
self.num_experts = self.num_local_experts * self.ep_size
self.routed_top_k = top_k
self.top_k = top_k + fuse_shared_expert
self.shared_expert_id = (
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
)
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.swiglu_limit = swiglu_limit or 0.0
@@ -58,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
self._runtime: Any = None
# (max_tok_per_rank, runtime), ascending by capacity.
self._runtimes: list[tuple[int, Any]] = []
def weight_loader(
self,
@@ -70,7 +122,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
return_success: bool = False,
) -> bool | None:
local_expert_id = expert_id - self.experts_start_idx
if not 0 <= local_expert_id < self.num_local_experts:
if not 0 <= local_expert_id < self.num_routed_local_experts:
return False if return_success else None
expert_data = param.data[local_expert_id]
if shard_id in ("w1", "w3"):
@@ -81,8 +133,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
expert_data.copy_(loaded_weight.view(torch.uint8))
return True if return_success else None
def load_shared_expert(self, shared_experts: nn.Module) -> None:
slot = self.num_routed_local_experts
for linear, weight, weight_scale in (
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
):
block_scale = get_fp8_block_weight_scale(linear)
assert block_scale is not None, "shared expert must be block-FP8"
group = GroupShape(
linear.weight.shape[0] // block_scale.shape[0],
linear.weight.shape[1] // block_scale.shape[1],
)
packed, scale = mxfp4_quantize(
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
)
weight.data[slot].copy_(packed.view(torch.uint8))
weight_scale.data[slot].copy_(scale.view(torch.uint8))
def finalize_weights(self) -> None:
if self._runtime is not None:
if self._runtimes:
return
n = self.num_local_experts
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
@@ -97,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
)
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
FIXED_SLOT_MAX_MTPR,
)
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
# must also be a power of two.
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
capacities = [self.max_num_tokens]
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
fixed_slot_max_tokens
):
capacities.insert(0, fixed_slot_max_tokens)
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
build_mega_moe_bundle_plan,
)
return build_mega_moe_bundle_plan(
max_tok_per_rank,
experts_per_rank=self.num_local_experts,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
world_size=self.ep_size,
).fixed_slot_dispatch
def get_runtime(self, max_tok_per_rank: int) -> Any:
key = (
id(get_ep_group().cpu_group),
torch.accelerator.current_device_index(),
@@ -104,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.top_k,
self.hidden_size,
self.intermediate_size,
self.max_num_tokens,
max_tok_per_rank,
self.swiglu_limit,
)
runtime = self._runtime_cache.get(key)
@@ -123,11 +222,53 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
w1_scale=self.w13_weight_scale,
w2=self.w2_weight,
w2_scale=self.w2_weight_scale,
max_tok_per_rank=self.max_num_tokens,
max_tok_per_rank=max_tok_per_rank,
swiglu_limit=self.swiglu_limit,
)
self._runtime_cache[key] = runtime
self._runtime = runtime
return runtime
def select_runtime(self, num_tokens: int) -> Any:
# Dispatch is collective, so every EP rank must pick the same runtime.
if is_forward_context_available():
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is not None:
num_tokens = max(
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
)
for max_tok_per_rank, runtime in self._runtimes:
if num_tokens <= max_tok_per_rank:
return runtime
raise ValueError(
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
)
def append_shared_expert(
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Remap routed ids to the per-rank (routed + shared) layout and append
a weight-1.0 route to this rank's shared slot."""
num_tokens = topk_ids.shape[0]
out_ids = torch.empty(
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
)
out_weights = torch.empty(
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
)
if num_tokens:
block_m = 64
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
topk_ids.contiguous(),
topk_weights.to(torch.float32).contiguous(),
out_ids,
out_weights,
num_tokens,
self.shared_expert_id,
TOPK=self.routed_top_k,
ROUTED_PER_RANK=self.num_routed_local_experts,
BLOCK_M=block_m,
)
return out_weights, out_ids
def forward(
self,
@@ -135,8 +276,10 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
runtime = self._runtime
assert runtime is not None
assert self._runtimes
runtime = self.select_runtime(hidden_states.shape[0])
if self.fuse_shared_expert:
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
runtime._s1_w1 = self.w13_weight
runtime._s1_w1_scale = self.w13_weight_scale
@@ -149,6 +292,10 @@ DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type:
def finalize_mega_moe_weights(model: nn.Module) -> None:
for module in model.modules():
if isinstance(module, DeepseekV4AiterMegaMoEExperts):
module.finalize_weights()
for module in list(model.modules()):
experts = getattr(module, "experts", None)
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
if experts.fuse_shared_expert:
experts.load_shared_expert(module.shared_experts)
module.shared_experts = None
experts.finalize_weights()
+22 -21
View File
@@ -74,11 +74,15 @@ from vllm.model_executor.models.utils import (
make_layers,
maybe_prefix,
)
from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts
from vllm.models.deepseek_v4.amd.mega_moe import (
DeepseekV4AiterMegaMoEExperts,
finalize_mega_moe_weights,
)
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
@@ -604,7 +608,10 @@ class DeepseekV4MoE(nn.Module):
self.image_sentinel_lo = (
IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0
)
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
num_hash_layers = getattr(config, "num_hash_layers", None) # Set by V4.1
if num_hash_layers is None:
num_hash_layers = config.mlp_layer_types.count("hash_moe")
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
@@ -690,6 +697,10 @@ class DeepseekV4MoE(nn.Module):
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
swiglu_limit=self.swiglu_limit,
fuse_shared_expert=(
config.n_shared_experts == 1
and envs.VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS
),
prefix=f"{prefix}.experts",
)
return
@@ -1221,7 +1232,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if self._mtp_hidden_buffer is not None:
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
# Each DBO ubatch owns a different slice of the batch. Writing
# from row 0 in both threads would let one ubatch clobber the
# other's rows in the buffer the MTP drafter reads after the step.
offset = dbo_current_ubatch_token_offset()
self._mtp_hidden_buffer[offset : offset + num_tokens].copy_(
hidden_states.flatten(1)
)
hidden_states = self.hc_head_op(
hidden_states,
@@ -1258,14 +1275,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1355,14 +1364,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
@@ -1524,6 +1525,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
# Must precede the gate_up preshuffle of the fused shared experts.
finalize_mega_moe_weights(self)
# After per-layer quant finalize, so we preshuffle the final fp8 weights.
fused_compressor_layers = 0
for module in self.modules():
@@ -1532,8 +1535,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
module.prepare_attn_preshuffle()
elif isinstance(module, DeepseekV4MLP):
module.prepare_gateup_preshuffle()
elif isinstance(module, DeepseekV4AiterMegaMoEExperts):
module.finalize_weights()
if fused_compressor_layers:
logger.info(
"Fused the C4 compressor GEMMs in %d DeepSeek V4 layers",
+7 -19
View File
@@ -19,10 +19,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -43,11 +39,12 @@ from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.models.deepseek_v4.quant_config import align_mtp_quant_config
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .mega_moe import finalize_mega_moe_weights
from .model import DeepseekV4DecoderLayer
from .model import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM
logger = init_logger(__name__)
@@ -293,10 +290,15 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
class DeepSeekV4MTP(nn.Module):
# The draft head reuses DeepseekV4DecoderLayer, so its fused linears must
# unfold to the same checkpoint shards for per-layer quant config lookup.
packed_modules_mapping = DeepseekV4ForCausalLM.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
align_mtp_quant_config(self.quant_config, self.config)
self.model = DeepSeekV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
@@ -384,14 +386,6 @@ class DeepSeekV4MTP(nn.Module):
# KeyError downstream rather than be silently rewritten.
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
expert_mapping = fused_moe_make_expert_params_mapping(
@@ -489,12 +483,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
+14
View File
@@ -162,6 +162,16 @@ def apply_pre_quantized_block_scaled_mm(
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
def _default_first_token_threshold() -> int:
"""Max tokens at which the CSA/HCA fork enqueues the default chain first.
At small batches starting the long default chain first shortens the step
tail; larger batches saturate the device and aux-first is faster. Only
tuned on gfx950 (24 tokens = conc 8 with MTP=2); 0 disables elsewhere.
"""
return 24 if _ON_GFX950 else 0
@triton.jit
def _combine_topk_swa_indices_kernel(
combined_indices_ptr,
@@ -807,6 +817,8 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
indexer = self.indexer
compressor = self.compressor
assert compressor is not None
# Static per captured graph: each capture size bakes its own order.
default_first = hidden_states.shape[0] <= _default_first_token_threshold()
def default_chain():
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
@@ -838,6 +850,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
[self.ln_events[1]],
aux_streams[:1],
enable=True,
default_first=default_first,
)
self._sparse_indexer_and_attn(
hidden_states, None, None, None, q, kv_out, positions, o_padded
@@ -864,6 +877,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
self.ln_events[1:3],
aux_streams[:2],
enable=True,
default_first=default_first,
)
indexer_weights_out, _ = indexer.weights_proj(hidden_states)
+4 -5
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.models.common.ops.fused_qk_rmsnorm import (
_FUSED_Q_KV_RMSNORM_KERNEL,
fused_q_kv_rmsnorm,
@@ -230,7 +231,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# we do this for because MTP layer is not included
# in the compress ratio list
if layer_id < config.num_hidden_layers:
self.compress_ratio = max(1, config.compress_ratios[layer_id])
layer_type = config.layer_types[layer_id]
self.compress_ratio = max(1, config.compress_rates.get(layer_type, 0))
else:
self.compress_ratio = 1
self.eps = config.rms_norm_eps
@@ -240,10 +242,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
# Sink padded to the same head count, initialized to -inf (no sink
# effect). Weight loading fills the first n_local_heads slots.
self.attn_sink = nn.Parameter(
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
requires_grad=False,
)
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
self.fused_wqa_wkv = MergedColumnParallelLinear(
self.hidden_size,
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared EPLB MoE registration helpers for DeepSeek V4 family models."""
from collections.abc import Iterable
from typing import Any
import torch.nn as nn
from vllm.config import ModelConfig
def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool:
"""Return whether a DSpark draft can share EPLB state with the target.
Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts
use a smaller routed-expert count and cannot share EPLB state.
"""
return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4"
def collect_moe_layers(
moe_model: Any,
layers: Iterable[nn.Module],
config: object,
*,
decoder_layer_type: type[nn.Module],
moe_type: type[nn.Module],
) -> None:
"""Populate ``MixtureOfExperts`` fields from decoder layers for EPLB.
Works for both target backbones and DSpark draft stacks. Draft models pass
their shorter ``model.layers`` list; pipeline-parallel placeholder layers
are skipped via ``decoder_layer_type``.
"""
moe_model.num_expert_groups = getattr(config, "n_group", 1)
moe_model.moe_layers = []
moe_model.moe_mlp_layers = []
example_moe = None
for layer in layers:
if not isinstance(layer, decoder_layer_type):
continue
if not isinstance(layer.ffn, moe_type):
continue
example_moe = layer.ffn
moe_model.moe_mlp_layers.append(layer.ffn)
moe_model.moe_layers.append(layer.ffn.experts)
moe_model.num_moe_layers = len(moe_model.moe_layers)
moe_model.extract_moe_parameters(example_moe)
@@ -27,7 +27,7 @@ from typing import Any, assert_never
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature
from transformers import BatchFeature, DeepseekV4Config
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
@@ -48,7 +48,6 @@ from vllm.multimodal.processing.processor import (
UpdateMode,
_plan_prompt_updates,
)
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
COMPRESS_PAD_TO = 4
@@ -34,6 +34,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
triton_scalar_specialization_rep,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import next_power_of_2
@@ -393,7 +394,9 @@ class DequantizeAndGatherKCacheKernel(
compress_ratios = frozenset(
max(1, int(compress_ratio))
for compress_ratio in vllm_config.model_config.hf_config.compress_ratios
for compress_ratio in get_compress_ratios(
vllm_config.model_config.hf_config
)
)
return self._trace_dispatch(self.dispatch)(
zip_inputs(
@@ -658,7 +661,7 @@ class ComputeGlobalTopkIndicesAndLensKernel(
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
index_topk = vllm_config.model_config.hf_config.index_topk
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
cache_block_size = vllm_config.cache_config.block_size
if index_topk <= 0 or cache_block_size <= 0:
return []
@@ -964,7 +967,7 @@ class CombineTopkSwaIndicesKernel(
hf_config = vllm_config.model_config.hf_config
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
compress_ratios = tuple(
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
)
max_c128a_topk = (
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
@@ -1592,7 +1595,7 @@ class BuildFlashinferMixedSparseIndicesKernel(
image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
compress_ratios = tuple(
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
)
max_c128a_topk = (
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
@@ -33,7 +33,9 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.math_utils import round_up
from vllm.v1.attention.ops.common import pack_ue8m0_x4
if current_platform.is_rocm():
from vllm.platforms.rocm import _ON_GFX950
@@ -742,6 +744,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
TOKEN_STRIDE: tl.constexpr, # 128 for indexer
SCALE_DIM: tl.constexpr, # 4 for indexer (1 float32)
KV_BLOCK_STRIDE: tl.constexpr,
MX_SCALE: tl.constexpr, # store the scale as 4 packed UE8M0 bytes
):
"""Fused compress → RMSNorm → RoPE → FP8 quant → store.
@@ -878,7 +881,10 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
# Single float32 scale
scale_val = tl.exp2(exponent)
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
if MX_SCALE:
tl.store(scale_ptr.to(tl.pointer_type(tl.int32)), pack_ue8m0_x4(scale_val))
else:
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
# =============================================================================
@@ -927,6 +933,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
TOKEN_STRIDE: tl.constexpr, # HEAD_SIZE // 2 = 64 packed bytes/token
SCALE_DIM: tl.constexpr, # HEAD_SIZE // QUANT_BLOCK = 4 ue8m0 bytes/token
KV_BLOCK_STRIDE: tl.constexpr,
MX_SCALE: tl.constexpr, # unused for MXFP4 (kept for signature parity)
):
"""Fused compress → RMSNorm → RoPE → MXFP4 quant → store.
@@ -1102,6 +1109,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
block_size: int
kv_cache_block_size: int
kv_block_stride: int
mx_scale: bool
# Canonical kernel for the launcher's arg_names mapping; the launched
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
@@ -1181,6 +1189,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
block_size=state_block_size,
kv_cache_block_size=kv_cache_block_size,
kv_block_stride=kv_block_stride,
mx_scale=not use_fp4_cache and mqa_logits_use_mx_scales(),
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
@@ -1301,6 +1310,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
TOKEN_STRIDE=token_stride,
SCALE_DIM=scale_dim,
KV_BLOCK_STRIDE=kv_cache.stride(0),
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
num_warps=1,
**pdl_kwargs,
)
@@ -40,6 +40,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItem
from .mm_preprocess import (
COMPRESS_PAD_TO,
IMAGE_PLACEHOLDER,
IMAGE_SENTINEL_BASE_ID,
DeepseekV4VLDummyInputsBuilder,
@@ -105,6 +106,10 @@ class DeepseekV4ForConditionalGeneration(
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
requires_raw_input_tokens = True
supports_tower_connector_lora = True
# Image spans widen the sparse-SWA window in-kernel and the placeholder
# starts with a compressor-alignment pad
mm_prefix_clamp_sliding_window = True
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+3 -3
View File
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, cast
import torch
from torch import nn
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -409,12 +409,12 @@ class DeepseekCompressor(nn.Module):
pdl_kwargs=pdl_kwargs,
)
# full graph cannot branch on per-step CPU metadata after capture
# A captured graph would replay the capture batch's skip decision.
if (
current_platform.is_cuda()
and self.head_dim == 512
and self.compress_ratio == 128
and forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL
and not torch.cuda.is_current_stream_capturing()
and state_metadata.c128_boundary is False
):
return
+3 -17
View File
@@ -145,7 +145,9 @@ class DeepseekV4MoE(nn.Module):
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
"hash_moe"
)
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
@@ -663,14 +665,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -755,14 +749,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
+23 -25
View File
@@ -18,10 +18,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -46,17 +42,20 @@ from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4MixtureOfExperts,
DeepseekV4Model,
DeepseekV4MoE,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
@@ -346,7 +345,7 @@ def _insert_context_kv(
)
class DSparkDeepseekV4ForCausalLM(nn.Module):
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
# load_dspark_model always aliases the target's.
has_own_embed_tokens = False
@@ -373,6 +372,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
collect_moe_layers(
self,
self.model.layers,
self.config,
decoder_layer_type=DeepseekV4DecoderLayer,
moe_type=DeepseekV4MoE,
)
# --- Hooks used by the speculator -------------------------------------
@@ -451,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
num_redundant_experts=first_layer.ffn.n_redundant_experts,
)
expert_scale_suffix = (
".weight_scale"
@@ -470,12 +480,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
@@ -493,6 +497,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -544,11 +555,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
@@ -560,17 +566,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
self.process_weights_after_loading()
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def _finalize_moe(self) -> None:
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
+101 -262
View File
@@ -14,7 +14,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_ep_group,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.distributed.eplb.eplb_state import EplbLayerState
@@ -33,9 +32,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmMegaMoEBackend,
get_deep_gemm_mega_moe_backend,
ue8m0_uint8_to_float,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
eplb_map_to_physical_and_record,
@@ -81,6 +78,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_shard,
)
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
DeepseekV4FlashInferMLAAttention,
DeepseekV4FlashInferSM120Attention,
@@ -224,6 +222,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
hidden_size: int,
intermediate_size: int,
num_shared_experts: int = 0,
shared_experts: nn.Module | None = None,
prefix: str = "",
num_logical_experts: int | None = None,
):
@@ -237,7 +236,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_shared_experts = num_shared_experts
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.num_logical_experts = (
@@ -246,64 +244,33 @@ class DeepseekV4MegaMoEExperts(nn.Module):
self.eplb_state = EplbLayerState()
weight_attrs = {"weight_loader": self.weight_loader}
self.w13_weight = nn.Parameter(
torch.zeros(
num_local_experts,
2 * intermediate_size,
hidden_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
device = torch.get_default_device()
if device.type != current_platform.device_type: # meta under ipc_cache
device = torch.device(
current_platform.device_type, torch.accelerator.current_device_index()
)
self.backend = get_deep_gemm_mega_moe_backend(
device, hidden_size, intermediate_size
)
set_weight_attrs(self.w13_weight, weight_attrs)
self.w13_weight_scale = nn.Parameter(
torch.zeros(
num_local_experts,
2 * intermediate_size,
hidden_size // 32,
dtype=torch.uint8,
),
requires_grad=False,
if num_shared_experts and not self.backend.supports_shared_experts(
shared_experts
):
num_shared_experts = 0
self.num_shared_experts = num_shared_experts
self.backend.create_weights(
self,
num_local_experts=num_local_experts,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
num_shared_experts=num_shared_experts,
)
set_weight_attrs(self.w13_weight_scale, weight_attrs)
for name, param in self.named_parameters(recurse=False):
shared = name.startswith("shared_")
loader = self.shared_weight_loader if shared else self.weight_loader
set_weight_attrs(param, {"weight_loader": loader})
self.w13_weight_scale.quant_method = "block"
self.w2_weight = nn.Parameter(
torch.zeros(
num_local_experts,
hidden_size,
intermediate_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
set_weight_attrs(self.w2_weight, weight_attrs)
self.w2_weight_scale = nn.Parameter(
torch.zeros(
num_local_experts,
hidden_size,
intermediate_size // 32,
dtype=torch.uint8,
),
requires_grad=False,
)
set_weight_attrs(self.w2_weight_scale, weight_attrs)
self.w2_weight_scale.quant_method = "block"
self._transformed_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None
self._transformed_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None
self._transformed_shared_l1_weights: (
tuple[torch.Tensor, torch.Tensor] | None
) = None
self._transformed_shared_l2_weights: (
tuple[torch.Tensor, torch.Tensor] | None
) = None
self._backend: DeepGemmMegaMoEBackend | None = None
# Register in the static forward context so the custom-op wrapper
# can look up this module by name from within a torch.compile graph.
compilation_config = vllm_config.compilation_config
@@ -331,144 +298,57 @@ class DeepseekV4MegaMoEExperts(nn.Module):
return_success: bool = False,
) -> bool | None:
local_expert_ids = self._map_global_expert_id(expert_id)
if not local_expert_ids:
param_prefix = "w2_" if shard_id == "w2" else "w13_"
if not local_expert_ids or param_prefix not in weight_name:
return False if return_success else None
loaded_any = False
is_scale = weight_name.endswith("weight_scale")
for local_expert_id in local_expert_ids:
expert_data = param.data[local_expert_id]
if shard_id in ("w1", "w3"):
if "w13_" not in weight_name:
continue
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
expert_data = expert_data.narrow(
0, shard_offset, self.intermediate_size
)
elif shard_id == "w2":
if "w2_" not in weight_name:
continue
else:
raise ValueError(f"Unsupported expert shard id: {shard_id}")
if expert_data.shape != loaded_weight.shape:
raise ValueError(
f"DeepSeek V4 MegaMoE expert weight shape mismatch for "
f"{weight_name}: parameter shard {tuple(expert_data.shape)} "
f"vs checkpoint {tuple(loaded_weight.shape)}"
)
expert_data.copy_(loaded_weight)
loaded_any = True
if return_success:
return loaded_any
return None
@staticmethod
def _ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
return ue8m0_uint8_to_float(sf)
def _ensure_backend(self) -> DeepGemmMegaMoEBackend:
"""Lazily resolve the MegaMoE backend.
The backend is looked up via :func:`get_deep_gemm_mega_moe_backend`, which
performs all runtime capability checks. Failure propagates directly
to the caller. This must be called before :meth:`finalize_weights`
releases the loader-side parameters.
"""
if self._backend is not None:
return self._backend
assert self.w13_weight is not None, (
"_ensure_backend() must be called before finalize_weights() "
"releases the loader-side parameters."
)
self._backend = get_deep_gemm_mega_moe_backend(
self.w13_weight.device,
self.hidden_size,
self.intermediate_size,
)
return self._backend
def _check_runtime_supported(self) -> None:
"""Preserved for API compatibility; delegates to the backend check."""
self._ensure_backend()
def _finalize_shared_expert_weights(self, shared_experts: DeepseekV4MLP) -> None:
"""Delegate shared-expert weight transform to the backend and update state.
On success, populates ``self._transformed_shared_l{1,2}_weights``. If
the backend cannot fuse shared experts (returns ``None``), disables
native fusion by setting ``self.num_shared_experts = 0``.
"""
backend = self._ensure_backend()
result = backend.transform_shared_expert_weights(
shared_experts=shared_experts,
num_shared_experts=self.num_shared_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
prefix=self.prefix,
)
if result is None:
self.num_shared_experts = 0
return
(
self._transformed_shared_l1_weights,
self._transformed_shared_l2_weights,
) = result
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
backend = self._ensure_backend()
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
if self._transformed_l1_weights is None:
self._transformed_l1_weights, self._transformed_l2_weights = (
backend.transform_weights(
w13_weight=self.w13_weight.data,
w13_weight_scale=self.w13_weight_scale.data,
w2_weight=self.w2_weight.data,
w2_weight_scale=self.w2_weight_scale.data,
num_local_experts=self.num_local_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
)
self.backend.load_weight(
param.data[local_expert_id], loaded_weight, shard_id, is_scale
)
# Drop the original loader-side parameters: the MegaMoE kernels only
# consume the transformed views above. transform_weights_for_mega_moe
# allocates a fresh tensor for the L1 weight (see
# _interleave_l1_weights) and fresh SF tensors for L1/L2; the L2
# weight is the only tensor that aliases the original storage, and
# _transformed_l2_weights still holds it, so the storage stays live
# after we drop the Parameter.
self.w13_weight = None
self.w13_weight_scale = None
self.w2_weight = None
self.w2_weight_scale = None
return True if return_success else None
if shared_experts is None or self.num_shared_experts == 0:
return
if self._transformed_shared_l1_weights is not None:
return
if not backend.supports_shared_experts(deep_gemm):
logger.warning_once(
"Disabling native MegaMoE shared-expert fusion because the "
"installed DeepGEMM Python API is older than the vLLM "
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
)
self.num_shared_experts = 0
return
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
"""Load one shared-expert checkpoint tensor; return the parameter name."""
if ".w2." in name or ".down_proj." in name:
param_name = "shared_l2_weight"
elif ".w1." in name or ".w3." in name:
param_name = "shared_l1_weight"
else:
raise ValueError(f"Unexpected shared-expert tensor: {name}")
if "weight_scale" in name:
param_name += "_scale"
param = getattr(self, param_name)
param.weight_loader(param, loaded_weight, name)
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
self._finalize_shared_expert_weights(shared_experts)
def shared_weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
) -> None:
shard_id = (
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
)
is_scale = "weight_scale" in weight_name
self.backend.load_weight(param.data, loaded_weight, shard_id, is_scale)
@property
def _transformed_l1_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
return self.w13_weight, self.w13_weight_scale
@property
def _transformed_l2_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
return self.w2_weight, self.w2_weight_scale
@property
def has_fused_shared_experts(self) -> bool:
return self._transformed_shared_l1_weights is not None
return self.num_shared_experts > 0
def get_symm_buffer(self):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
backend = self._ensure_backend()
backend = self.backend
group = get_ep_group().device_group
device = torch.accelerator.current_device_index()
key = (
@@ -514,10 +394,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
)
def get_expert_weights(self) -> list[torch.Tensor]:
self.finalize_weights()
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
assert t.shape[0] == self.num_local_experts
@@ -600,7 +476,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
else None,
)
backend = self._ensure_backend()
backend = self.backend
symm_buffer = self.get_symm_buffer()
shared_x_sf = None
shared_block_m = None
@@ -629,28 +505,21 @@ class DeepseekV4MegaMoEExperts(nn.Module):
hidden_quant=backend.hidden_quant,
)
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
if self.has_fused_shared_experts:
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
shared_l1_weights=self._transformed_shared_l1_weights,
shared_l2_weights=self._transformed_shared_l2_weights,
activation_clamp=activation_clamp,
fast_math=fast_math,
)
else:
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
activation_clamp=activation_clamp,
fast_math=fast_math,
)
fused = self.has_fused_shared_experts
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
shared_l1_weights=(
(self.shared_l1_weight, self.shared_l1_weight_scale) if fused else None
),
shared_l2_weights=(
(self.shared_l2_weight, self.shared_l2_weight_scale) if fused else None
),
activation_clamp=activation_clamp,
fast_math=fast_math,
)
return y
@@ -841,9 +710,12 @@ class DeepseekV4MoE(nn.Module):
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
shared_experts=self.shared_experts,
prefix=f"{prefix}.experts",
)
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
if self.experts.has_fused_shared_experts:
self.shared_experts = None
def _init_fused_moe_experts(
self,
@@ -880,6 +752,7 @@ class DeepseekV4MoE(nn.Module):
renormalize=config.norm_topk_prob,
quant_config=quant_config,
prefix=f"{prefix}.experts",
ckpt_names=("w1", "w2", "w3"),
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
e_score_correction_bias=self.gate.e_score_correction_bias,
@@ -910,12 +783,7 @@ class DeepseekV4MoE(nn.Module):
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
# The MegaMoE experts module owns the backend; resolve and assert it.
self.experts._ensure_backend()
assert self.experts._backend is not None, (
"MegaMoE backend must be resolved before routing."
)
mega_backend = self.experts._backend
mega_backend = self.experts.backend
# Small local padded batches favor GateLinear; 128-expert gates cross earlier.
gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16
if (
@@ -982,10 +850,7 @@ class DeepseekV4MoE(nn.Module):
activation_clamp=activation_clamp,
)
if (
self.shared_experts is not None
and not self.experts.has_fused_shared_experts
):
if self.shared_experts is not None:
shared_output = self.shared_experts(hidden_states)
final_hidden_states += shared_output
@@ -1003,10 +868,6 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
def finalize_mega_moe_weights(self) -> None:
if self.use_native_mega_moe:
self.experts.finalize_weights(self.shared_experts)
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
"""Pick the CUDA sparse-MLA attention class for the configured backend.
@@ -1083,7 +944,7 @@ class DeepseekV4DecoderLayer(nn.Module):
vllm_config,
prefix=f"{prefix}.ffn",
use_sequence_parallel=self.use_sequence_parallel,
num_hash_layers=config.num_hash_layers,
num_hash_layers=config.mlp_layer_types.count("hash_moe"),
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -1346,7 +1207,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
prefix=f"{prefix}.layers",
)
self.has_local_hash_moe = self.start_layer < min(
self.end_layer, config.num_hash_layers
self.end_layer, config.mlp_layer_types.count("hash_moe")
)
if get_pp_group().is_last_rank:
@@ -1538,14 +1399,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1558,6 +1411,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
for name, loaded_weight in weights:
if ".shared_experts." in name:
if is_pp_missing_parameter(name, self):
continue
experts = self.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if pad_shared_expert and ".shared_experts." in name:
loaded_weight = self._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
@@ -1620,14 +1482,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
@@ -1687,10 +1541,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
num_experts=self.config.n_routed_experts,
)
def finalize_mega_moe_weights(self) -> None:
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1841,23 +1691,13 @@ class DeepseekV4ForCausalLM(
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
self.num_expert_groups = getattr(self.config, "n_group", 1)
self.num_moe_layers = self.config.num_hidden_layers
self.moe_layers: list[nn.Module] = []
self.moe_mlp_layers: list[DeepseekV4MoE] = []
example_moe: DeepseekV4MoE | None = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
if not isinstance(layer, DeepseekV4DecoderLayer):
continue
if isinstance(layer.ffn, DeepseekV4MoE):
example_moe = layer.ffn
self.moe_mlp_layers.append(layer.ffn)
self.moe_layers.append(layer.ffn.experts)
self.num_moe_layers = len(self.moe_layers)
self.extract_moe_parameters(example_moe)
collect_moe_layers(
self,
self.model.layers,
self.config,
decoder_layer_type=DeepseekV4DecoderLayer,
moe_type=DeepseekV4MoE,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -1900,7 +1740,6 @@ class DeepseekV4ForCausalLM(
return loaded_params
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+7 -26
View File
@@ -21,10 +21,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -386,14 +382,6 @@ class DeepSeekV4MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_native_mega_moe:
@@ -436,6 +424,13 @@ class DeepSeekV4MTP(nn.Module):
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[str(spec_layer)].mtp_block.ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -490,12 +485,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if name.endswith(".ffn.gate.bias"):
# ``e_score_correction_bias`` lives on the gate
@@ -529,17 +518,9 @@ class DeepSeekV4MTP(nn.Module):
f"Use a checkpoint that includes MTP layer weights, "
f"or disable speculative decoding."
)
self.process_weights_after_loading()
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def finalize_mega_moe_weights(self) -> None:
for layer in self.model.layers.values():
layer.mtp_block.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self.finalize_mega_moe_weights()
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
@@ -18,6 +18,7 @@ from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
class DequantGatherKCacheKernel(
@@ -329,7 +330,7 @@ class DequantGatherKCacheKernel(
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
block_size = vllm_config.cache_config.block_size
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
return self._trace_dispatch(self.dispatch)(
zip_inputs(
dict(
@@ -436,7 +436,7 @@ class SparseAttnCompressNormRopeStoreC4Kernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 4 in hf_config.compress_ratios
and "compressed_sparse_attention" in hf_config.layer_types
and cache_config.cache_dtype == "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
@@ -934,7 +934,7 @@ class SparseAttnCompressNormRopeStoreFullC4Kernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 4 in hf_config.compress_ratios
and "compressed_sparse_attention" in hf_config.layer_types
and cache_config.cache_dtype != "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
@@ -1366,7 +1366,7 @@ class SparseAttnCompressC128Block8Kernel(
return self._trace_dispatch(self.dispatch)(
head_size=hf_config.head_dim,
state_width=hf_config.head_dim,
enabled=128 in hf_config.compress_ratios,
enabled="heavily_compressed_attention" in hf_config.layer_types,
_when=lambda *, enabled: enabled,
)
@@ -2040,7 +2040,7 @@ class SparseAttnNormRopeStoreFullKernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 128 in hf_config.compress_ratios
and "heavily_compressed_attention" in hf_config.layer_types
and cache_config.cache_dtype != "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
+30
View File
@@ -21,11 +21,41 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
if TYPE_CHECKING:
from transformers import PreTrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
def align_mtp_quant_config(
quant_config: QuantizationConfig | None, config: PreTrainedConfig
) -> None:
"""Re-key the quant config's MTP entries onto the draft module prefixes.
``_make_deepseek_v4_weights_mapper`` renames the checkpoint's ``mtp.{i}.``
layers to ``model.mtp.{i}.``, but the draft modules are built under
``model.layers.{num_hidden_layers + i}.``. Without this remap no per-layer
entry matches, so every MTP linear falls back to the global scheme. On a
mixed checkpoint (FP8 attention and shared experts, NVFP4 routed experts)
that builds NVFP4 parameters for FP8 weights and the load then fails a
shape assertion.
"""
from vllm.model_executor.models.utils import WeightsMapper
if quant_config is None:
return
quant_config.apply_vllm_mapper(
WeightsMapper(
orig_to_new_prefix={
f"model.mtp.{i}.": f"model.layers.{config.num_hidden_layers + i}."
for i in range(config.num_nextn_predict_layers)
}
)
)
class DeepseekV4FP8Config(Fp8Config):
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
-17
View File
@@ -15,10 +15,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
@@ -322,12 +318,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -383,13 +373,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
if name not in params_dict:
continue
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
+3 -17
View File
@@ -638,7 +638,9 @@ class DeepseekV4MoE(nn.Module):
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
"hash_moe"
)
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
@@ -1149,14 +1151,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1219,14 +1213,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
-18
View File
@@ -19,10 +19,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -335,14 +331,6 @@ class DeepSeekV4MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_mega_moe:
@@ -441,12 +429,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
@@ -88,10 +88,13 @@ def _xpu_qnorm_rope_kernel(
kv_base = kv_ptr + token_idx * HEAD_DIM
kv_out_base = kv_out_ptr + token_idx * HEAD_DIM
# Copy full KV unchanged first
# Copy only the NoPE portion; the RoPE region is written exclusively
# by the rotated stores below. Masking avoids a write-after-write
# hazard on the overlapping RoPE addresses (matches the Q branch).
offs = tl.arange(0, HEAD_DIM)
kv_full = tl.load(kv_base + offs)
tl.store(kv_out_base + offs, kv_full)
nope_mask = offs < NOPE_DIM
tl.store(kv_out_base + offs, kv_full, mask=nope_mask)
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions
even_offs = NOPE_DIM + rope_pair_idx * 2
-15
View File
@@ -18,10 +18,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
@@ -420,12 +416,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -487,11 +477,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
+298 -37
View File
@@ -12,8 +12,8 @@ import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
@@ -58,8 +58,10 @@ from vllm.models.common.ops.sequence_parallel import (
from vllm.models.deepseek_v4.amd.model import (
DeepseekV4MoE as DeepseekV4MoEBase,
)
from vllm.models.deepseek_v41.amd.mono_decode import MonoDecodeLayer
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -255,6 +257,12 @@ class DeepseekV4DecoderLayer(nn.Module):
# attn_norm / ffn_norm into its collapse, so the separate norms are
# skipped for the seams it takes.
self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
# Decode steps of an eligible layer may run as the mono decode layer.
self.mono = MonoDecodeLayer.create(self, vllm_config)
if self.mono is not None and self.mono.ffn_only:
# A decode step hands wo_b's TP partial to the FFN launch; forward
# reduces it on the others.
self.attn.wo_b.reduce_results = False
@staticmethod
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
@@ -273,6 +281,20 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if self.mono is not None:
out = self.mono(
self,
x,
positions,
residual,
post_mix,
res_mix,
pre_mix,
engram_hashes,
engram_mask,
)
if out is not None:
return out
# Layer 0's attention seam projects the 2-D embedding with the folded
# hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn.
# The fused kernel only takes the latter, so that seam keeps the
@@ -371,6 +393,11 @@ class DeepseekV4DecoderLayer(nn.Module):
x = self.attn(positions, x, None)
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
elif self.mono is not None and self.mono.ffn_only:
out = self.mono.ffn(self, x, residual, post_mix, res_mix, attn_pre)
if out is not None:
return out
x = tensor_model_parallel_all_reduce(x)
residual, post_mix, res_mix, x, ffn_pre = self.mhc_pre_delayed(
residual,
@@ -394,6 +421,42 @@ class DeepseekV4DecoderLayer(nn.Module):
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix, ffn_pre
def write_kv(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
pre_mix: torch.Tensor,
post_mix: torch.Tensor,
res_mix: torch.Tensor,
residual: torch.Tensor,
) -> None:
"""Write the KV ``forward`` would for the replay layers' inputs, skipping
the rest of the layer; the replay batch then reruns its rows."""
# The replay batch rules out Engram and sequence parallel here.
assert self.engram is None and not self.use_sequence_parallel
fuse_attn_norm = self.fuse_seam_norm
residual, post_mix, res_mix, x, _attn_pre = self.mhc_pre_delayed(
residual,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
sublayer_out=x,
post_layer_mix=post_mix,
comb_res_mix=res_mix,
norm_weight=self.attn_norm.weight if fuse_attn_norm else None,
norm_eps=self.attn_norm.variance_epsilon,
)
if not fuse_attn_norm:
x = self.attn_norm(x)
self.attn.forward_kv(positions, x)
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -471,6 +534,45 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
prefix=f"{prefix}.layers",
)
# Config flag is the switch: ROCm forces swa_cache_layer.bounded_replay off.
self.decoder_replay_layers: DecoderReplayLayers | None = None
self.decoder_replay_start = self.end_layer
cut = max(config.kv_source_layer_ids)
if (
vllm_config.cache_config.swa_bounded_replay
and vllm_config.use_v2_model_runner
and cut < self.end_layer - 1
and self._decoder_replay_supported(vllm_config, cut)
):
assert self.layers[cut].attn.compress_ratio <= 1
# The last KV source writes every row's KV, then replays.
self.decoder_replay_start = cut
# The attention metadata keys the replay layers read.
metadata_prefixes: set[str] = set()
for layer in islice(self.layers, cut, self.end_layer):
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
metadata_prefixes.add(attn.swa_cache_layer.prefix)
if attn.compressed_cache_prefix is not None:
metadata_prefixes.add(attn.compressed_cache_prefix)
if attn.indexer is not None:
metadata_prefixes.add(attn.indexer.k_cache.prefix)
self.decoder_replay_layers = DecoderReplayLayers(
config.sliding_window,
self._run_replay_layers,
self.layers[cut].write_kv,
metadata_prefixes,
self.layers[cut].attn.swa_cache_layer.prefix,
)
logger.info_once(
"Decoder SWA bounded replay: in eager prefill steps, layer %d "
"writes its KV for every token, and the rest of it and layers "
"%d-%d run on each request's last %d tokens only.",
cut,
cut + 1,
self.end_layer - 1,
config.sliding_window,
)
# The n-gram hash needs a slot-keyed rolling store of compressed ids
# (chunked prefill / decode lookback); key it off the first local
# layer's sliding-window KV cache. Only PP ranks owning an engram
@@ -630,13 +732,33 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_first_rank:
assert intermediate_tensors is not None
pre_mix = intermediate_tensors["pre_mix"]
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
(
hidden_states,
residual,
post_mix,
res_mix,
pre_mix,
aux_hidden_states,
final_aux_recon,
ran_layers,
) = self._run_layers(
range(self.start_layer, self.decoder_replay_start),
hidden_states,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
engram_hashes,
engram_mask,
full_num_tokens,
)
# Replay collapses inside its own batch. The early loop still sets
# ran_layers, so that flag alone must not post again on the full batch.
collapsed = False
if self.decoder_replay_layers is not None:
hidden_states, pre_mix, *late_aux = self.decoder_replay_layers(
hidden_states,
positions,
input_ids,
@@ -644,20 +766,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
post_mix,
res_mix,
residual,
engram_hashes,
engram_mask,
)
if idx + 1 in self.aux_hidden_state_layers:
# Reconstruct the aux hidden state for draft models
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
aux_hidden_state = aux_recon.mean(dim=1)
if self.use_sequence_parallel:
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
final_aux_recon = aux_recon
if layer is not None:
# Reuse if the last layer was captured as an aux hidden state
if self.end_layer in self.aux_hidden_state_layers:
aux_hidden_states.extend(late_aux)
final_aux_recon = None
collapsed = True
if ran_layers and not collapsed:
if (
self.end_layer in self.aux_hidden_state_layers
and final_aux_recon is not None
):
hidden_states = final_aux_recon
else:
hidden_states = self.mhc_post(
@@ -687,6 +804,162 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def _run_layers(
self,
layer_ids: range,
hidden_states: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
pre_mix: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
residual: torch.Tensor | None,
engram_hashes: torch.Tensor | None,
engram_mask: torch.Tensor | None,
full_num_tokens: int,
) -> tuple[
torch.Tensor,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
list[torch.Tensor],
torch.Tensor | None,
bool,
]:
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None
ran_layers = False
for idx in layer_ids:
ran_layers = True
hidden_states, residual, post_mix, res_mix, pre_mix = self.layers[idx](
hidden_states,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
engram_hashes,
engram_mask,
)
# The aux id of the first replay layer is the post of the last
# full-batch layer. Capture it on the trimmed rows instead.
if idx + 1 in self.aux_hidden_state_layers and not (
self.decoder_replay_layers is not None
and idx + 1 == self.decoder_replay_start
):
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
aux_hidden_state = aux_recon.mean(dim=1)
if self.use_sequence_parallel:
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
final_aux_recon = aux_recon
return (
hidden_states,
residual,
post_mix,
res_mix,
pre_mix,
aux_hidden_states,
final_aux_recon,
ran_layers,
)
def _run_replay_layers(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
pre_mix: torch.Tensor,
post_mix: torch.Tensor,
res_mix: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, ...]:
"""Layers past the last KV source, on the rows they are given.
The final ``mhc_post`` runs here, on this batch. The return is
``(hidden_states, pre_mix, *aux)``.
"""
# Post of the layer before the cut. These rows are already trimmed,
# and sequence parallel is unsupported while replay is on.
cut_aux: list[torch.Tensor] = []
if self.decoder_replay_start in self.aux_hidden_state_layers:
cut_aux.append(
self.mhc_post(hidden_states, residual, post_mix, res_mix).mean(dim=1)
)
(
hidden_states,
residual,
post_mix,
res_mix,
pre_mix,
aux,
final_aux_recon,
ran_layers,
) = self._run_layers(
range(self.decoder_replay_start, self.end_layer),
hidden_states,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
None,
None,
positions.shape[0],
)
assert residual is not None and post_mix is not None
assert res_mix is not None and pre_mix is not None
if (
self.end_layer in self.aux_hidden_state_layers
and final_aux_recon is not None
):
hidden_states = final_aux_recon
elif ran_layers:
hidden_states = self.mhc_post(hidden_states, residual, post_mix, res_mix)
return (hidden_states, pre_mix, *cut_aux, *aux)
def _decoder_replay_supported(self, vllm_config: VllmConfig, cut: int) -> bool:
"""Whether this rank may replay from layer ``cut`` on; warns when not."""
parallel_config = vllm_config.parallel_config
spec_config = vllm_config.speculative_config
draft_config = spec_config.draft_model_config if spec_config else None
draft_hf_config = getattr(draft_config, "hf_config", None)
draft_window = getattr(draft_hf_config, "sliding_window", None)
draft_layer_types = getattr(draft_hf_config, "layer_types", None) or ()
window = self.config.sliding_window
if self.start_layer > cut or self.end_layer < self.config.num_hidden_layers:
reason = (
"the pipeline stage holding the last KV source layer must also "
"hold every layer after it"
)
elif (
self.use_sequence_parallel
or parallel_config.prefill_context_parallel_size > 1
or parallel_config.use_ubatching
):
reason = (
"the replay-layer batch shrinks per rank, which sequence and "
"prefill-context parallelism and microbatching cannot follow"
)
elif any(i >= cut for i in getattr(self.config, "engram_layer_ids", ())):
reason = "an Engram layer sits at or after the last KV source layer"
elif draft_config is not None and (
draft_window is None
or draft_window > window
or any(t != "sliding_attention" for t in draft_layer_types)
):
reason = (
f"the drafter (sliding window {draft_window}) reads hidden states "
f"outside the target's {window}-token window"
)
else:
return True
logger.warning_once("Decoder SWA bounded replay is off: %s.", reason)
return False
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
@@ -711,14 +984,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -799,14 +1064,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
@@ -1101,6 +1358,10 @@ class DeepseekV41LLMForCausalLM(
engram_hash = self.model.engram_hash
return engram_hash.lookback_depth if engram_hash is not None else 0
@property
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
return self.model.decoder_replay_layers
def forward(
self,
input_ids: torch.Tensor,
@@ -0,0 +1,97 @@
# DeepSeek-V4.1-Flash mono decode layer (gfx950, FlyDSL)
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
ATOM's V4.1 mono decode
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
from it, each file naming its origin; the attention stages (`attention/`) are
new, reading vLLM's KV records and weights as loaded.
## Scope
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
settled, without folding the previous output in, on layer 0 (the embedding
broadcast to the four streams) and after an Engram write (layers 1 and 14,
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
key table from that top-k. The FFN launch serves a layer outside these (an
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
The paths interleave freely (same tensors at the layer boundary).
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
routing bias is `e_score_correction_bias`).
- Contract: a drop-in for `DeepseekV4DecoderLayer.forward`:
in `x` (the previous FFN output, reduced, bf16 [M, 5120]), `residual`
(bf16 [M, 4, 5120]), `post_mix` [M, 4, 1], `res_mix` [M, 4, 4],
`pre_mix` [M, 4] (f32, the previous seam's)
out the MoE output (reduced), the residual after the FFN seam, and that
seam's post / comb / pre mixes.
## Launches
K1 seam (stages/seam.py): slice (160 x 32 columns): fold x into the
residual (post x + sum comb r), the collapsed input (pre . R'), partial
mixes (fn . R') and sums of squares -> gate (S): 24 mixes -> pre / post
/ comb (Sinkhorn 20) -> norm (S): attn_norm -> vLLM MXFP8 -> X8
attention front (attention/front.py): wqkv, q / kv norms + KV insert,
wq_b + RoPE -> Q
K2 attention back (attention/back.py): split attention, combine + inverse
RoPE, wo_a, wo_b; the bf16 wo_b partial pushed to every rank
FFN seam: reduce (the ranks' partials summed in rank order, fp32 ->
bf16: the attention output), slice, gate, norm -> the MoE input
MoE (stages/moe.py): xq, router (bf16 GEMV, f32 logits), route
(sqrtsoftplus + bias, top-6, renormalized x 1.5), shared expert (MXFP8
GEMVs, clamped SiLU), ug (MXFP4 x MXFP8 per expert of the step's
union), down (+ routing weight, top-k order; + shared) -> push, rank-order
sum -> out
FFN K2 without the attention back: push (160 x 32 columns) of vLLM's
unreduced wo_b output, as its bits, to every rank's attention region,
then K2's FFN seam and MoE (the same code, `layer.run_ffn`)
## Execution model and hand-offs
- Grid = 256 CTAs (one a CU, all resident), 512 threads a CTA, LDS a per-stage
union. A stage is a task table: CTA `b` runs tasks `b, b + 256, ...`, stages
in order; a consumer only waits on earlier stages, so no deadlock while the
grid is resident. A GEMV task issues its weight loads before it polls its
activations.
- In a launch: tagged pairs (value + tag in one 8 B store) or plain data +
flags, at device scope. The tag is the launch pair's epoch (K1 `2e + 1`,
K2 or the FFN launch `2e + 2`), a device word their CTA 0 moves on once every
CTA has marked the launch done: no clearing between steps, no host step hook, every argument
a fixed device pointer (graph-safe).
- Across ranks: one symmetric uncached buffer a rank (aiter `UncachedIpcHeap`),
every rank holding every peer's address; pushes are system-scope stores of
(value, tag) pairs. Every rank runs the same launches, so the epochs agree;
peer regions alternate by epoch parity, and a rank cannot run two steps ahead
of a peer (each step's reduce waits on every rank's push).
## Numerics (the vLLM path each stage reproduces)
- Seams: aiter `mhc_fused_post_pre_delayed_rmsnorm` (post fold order, fn as
bf16 hi + lo, products summed in order, Triton sigmoid / exp / rcp).
- Attention: vLLM's ROCm chain. `mxfp8(x)`: one E8M0 per 32 values,
`code = clamp(ceil(log2(amax / 448)) + 127, 0, 254)`; weights in vLLM's
row-major e4m3 [N, K] with [N/32, K/32] E8M0 scales (one shared copy); KV
records as vLLM's insert writes them (dims 0..447 e4m3 with a UE8M0 per 64,
RoPE dims bf16, 584 B); keys = the sliding window plus the top-k compressed
rows, softmax in base 2 with the attention sink folded in.
- MoE: vLLM's router (bf16 gate GEMV, f32 logits, `topk_hash_softplus_sqrt`)
and AITER a8w4 `fused_moe` (interleaved gate / up, swiglu limit 10, weights
in aiter's (16, 16) shuffle and `shuffle_scale` order); the shared expert as
vLLM's MXFP8 block-32 linears with `silu_and_mul_with_clamp`.
- All-reduces: exact, fp32 in rank order -> bf16. vLLM's decode all-reduces at
M <= 48 are below its INT4 quick-reduce threshold, so vLLM runs them exact too.
## Validation
- Per layer against vLLM's own modules in a TP process group (`DeepseekV4MoE`,
the mHC seams, the attention chain, the all-reduce), real checkpoint weights.
- Bench: layers of one type back to back in one HIP graph, cold weights,
us per layer, M = 1..48, TP2 / TP4; then e2e (AgentX, InferenceX#3690
recipe).
@@ -0,0 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1-Flash mono decode layer: one decoder layer of a decode step in
two persistent FlyDSL launches, its TP all-reduces in-kernel (README.md). The
host side is ``runner``; vLLM docks it in ``..mono_decode``."""
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The mono layer's attention stages: ``front`` (K1: wqkv, q / kv norms and the
KV insert, wq_b) and ``back`` (K2: split attention, combine, wo_a, wo_b)."""
@@ -0,0 +1,888 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""K2 ``back``: one DeepSeek-V4.1 attention layer from the rotated query to the
rank's wo_b partial.
One launch a layer and rank, ``BLOCKS`` x ``THREADS``, S rows:
split (S x chunks of CK keys): flash-decoding over one chunk of a token's
keys (K1's key table: top-k compressed rows, then window rows) for
every head of the rank. The token's q in LDS; each wave one
(head tile, 16-key tile): its fp8_ds_mla rows loaded once,
dequantized into its QK operands and kept raw in LDS; QK (bf16
MFMA), the chunk's max / sum per head, bf16 p in LDS; PV over the
chunk's raw rows (wave w: dims 64 w ..) -> the chunk's partial
(bf16) and its (max, sum) per head, published
combine (S x H): a (token, head)'s chunks merged with the sink folded in,
bf16, inverse GPT-J RoPE of dims 448.., vLLM's MXFP8 of each 32
-> XO (published)
wo_a (G x 1024 / 32, two 16-row tiles): the group's XO rows times wo_a
-> bf16 -> MXFP8 of the 32 rows -> X8B (published)
wo_b (320, 16 rows): X8B times wo_b -> bf16 -> ``out``
Hand-offs are plain device-scope data behind tagged flags (half the bytes of a
tagged pair per value). Adapted from ATOM's V4.1 ``attn_post`` (K2a), which
reads a bf16 pool and ends in a peer all-reduce.
"""
from dataclasses import dataclass
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T, as_ir_value
from .device import (
BLOCKS,
CM_DEV,
THREADS,
WAVES,
Mailbox,
bf16_round,
bf_hi,
bf_lo,
butterfly,
clamp_fp8,
first_task,
fp8_pack4,
fp8x8_bf16,
fresh,
gload,
gstore,
hw_exp2,
lane_gather,
ld_f32,
ld_i32,
mfma_bf16,
mx_code,
mx_mul,
pow2,
rsrc,
spin_until,
stamp,
traced,
xshfl,
)
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
from .plan import (
DATA,
HALF,
HEAD_DIM,
HEAD_TILE,
KEYS,
NOPE,
ROPE,
ROWS,
WOA_ROWS,
Dims,
back_scratch,
cdiv,
front_end,
tile_rows,
token_tiles,
)
_STREAM = fx.Stream(None)
NEG = -3.4028234663852886e38
LOG2E = 1.4426950408889634
REC = DATA + 8 # a raw record row in LDS: 576 data bytes + 8 scale bytes
Q_ROW = (
HEAD_DIM // 2 + 4
) # a q row in LDS (i32 words): 512 bf16 + 16 B, no bank aliasing
P_PAD = 2 # i32 words past a P row: 16 head rows read together hit distinct banks
# wo_a's K order: waves 0..3 / 4..7 take the two 16-row tiles, a contiguous K
# range a wave, summed in wave order
WOA_SPLIT_K = WAVES * ROWS // WOA_ROWS
BACK_POINTS = 16 # start, stage ends, split internals, GEMV internals
@dataclass(frozen=True)
class BackBuild:
tokens: int
tp: int
ratio: int # 0: window only; 1 / 2: top-k over a compressed cache
timeline: bool = False
def plus(t0, v):
return v if isinstance(t0, int) and t0 == 0 else v + t0
def head_groups(d: Dims, s: int) -> int:
"""Wave groups a split unit's head tiles are spread over: two (a tile each,
64-key chunks, twice the units) while the units fit the grid, else one (a
wave's keys loaded and dequantized once for every head tile)."""
return 2 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= BLOCKS else 1
def chunk_keys(d: Dims, s: int) -> int:
"""A split unit's keys: a 16-key tile a wave of a head group."""
return 16 * (WAVES // head_groups(d, s))
def p_row(d: Dims, s: int) -> int:
"""A P row (one head's chunk of bf16 p) in LDS, i32 words."""
return chunk_keys(d, s) // 2 + P_PAD
@traced
def publish(c, region, i):
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if c["tid"] == 0:
c["mb"].put(region, i, fx.Int32(1))
@traced
def await_flags(c, region, n, index):
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
tid = c["tid"]
if tid < n:
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
gpu.barrier()
# ---------------------------------------------------------------- keys
def key_rec(c, t, k):
"""(data address, scale address, valid) of token t's key k from K1's key
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
none (it reads a zero record: its scores are masked, its values zeros)."""
a, ratio = c["args"], c["ratio"]
# a table this launch wrote (``krdy``) is read past the CU's cache
cm = CM_DEV if const_expr("krdy" in c) else 0
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
ok = val != fx.Int32(-1)
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
if const_expr(ratio > 0):
is_c = ok & (val < 0)
blk = is_c.select(a["comp_block"], a["swa_block"])
stride = is_c.select(a["comp_stride"], a["swa_stride"])
base = is_c.select(a["comp"], a["swa"])
else:
blk, stride, base = a["swa_block"], a["swa_stride"], a["swa"]
b = slot // blk
off = slot % blk
page = base + fx.Int64(b) * stride
z = a["zrec"]
data = ok.select(page + fx.Int64(off * DATA), z)
sc = ok.select(page + fx.Int64(blk * DATA + off * 8), z + fx.Int64(DATA))
return data, sc, ok
def _scale_byte(scw, blk64):
return (scw[blk64 // 4] >> (blk64 % 4 * 8)) & 0xFF
def _fp8_pair_f32(word, scale, hi):
v = rocdl.cvt_scalef32_pk_f32_fp8(
fx.Vector.make_type(2, fx.Float32),
as_ir_value(fx.Int32(word)),
as_ir_value(fx.Float32(scale)),
hi,
)
v = fx.Vector(v)
return [v[0], v[1]]
# ---------------------------------------------------------------- split
@traced
def stage_split(c, unit):
"""Flash-decoding over chunk ``unit % nchunk`` of token ``unit // nchunk``,
every head of the rank -> the chunk's partial and (max, sum) per head. Wave
(head group hg, key tile kt): keys 16 kt .. of the chunk (loaded and
dequantized once) against the group's head tiles; then dims 64 w .. of the
PV over every key and head."""
tid, lane, wave, d, s = c["tid"], c["lane"], c["wave"], c["d"], c["S"]
a, sl = c["args"], c["sl"]
nchunk = c["nchunk"]
t = unit // nchunk
ch = unit % nchunk
ck = chunk_keys(d, s)
ngrp = head_groups(d, s)
nkt = WAVES // ngrp # 16-key tiles a chunk
nht = d.head_tiles // ngrp # head tiles a wave
hgi = wave // nkt
kt = wave % nkt
g = lane // 16
r16 = lane % 16
k0 = ch * ck
if const_expr("krdy" in c):
await_flags(c, c["krdy"], 1, lambda u: t)
# the key count, this lane's key and q: independent loads, out together
# (the key table holds -1 past a token's keys)
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
key = k0 + kt * 16 + r16
data, sc, okv = key_rec(c, t, key)
qrow = d.heads * HEAD_DIM // 8
qv = []
for i in range_constexpr(cdiv(qrow, THREADS)):
u = fresh(tid) + THREADS * i
qv.append(
fx.Vector(
bo.buffer_load(
rsrc(a["q"]),
(t * d.q_rows) // 2 + fx.min(u, qrow - 1) * 4,
vec_width=4,
dtype=T.i32,
)
)
)
if k0 < kv_len:
# a key's 448 e4m3 as 7 x 16 B (lane g: dims 64 c + 16 g ..), its 64
# bf16 rope dims as 2 x 16 B (32 c + 8 g ..), its 7 scale bytes
scw = gload(sc, words=2)
nope = [
gload(data + fx.Int64(64 * c4 + 16 * g), words=4)
for c4 in range(NOPE // 64)
]
rope = [
gload(data + fx.Int64(NOPE + (cc * 32 + 8 * g) * 2), words=4)
for cc in range(ROPE // 32)
]
# q of every head into LDS (bf16 rows of 512 + pad), 16 B a thread
for i in range_constexpr(cdiv(qrow, THREADS)):
u = fresh(tid) + THREADS * i
if u < qrow:
fx.ptr_store(
qv[i],
sl["q"]
+ (u // (HEAD_DIM // 8)) * Q_ROW
+ (u % (HEAD_DIM // 8)) * 4,
)
# the raw rows to LDS for the PV (head group 0's waves)
if hgi == 0:
rawb = sl["raw"] + (kt * 16 + r16) * (REC // 4)
for c4 in range_constexpr(NOPE // 64):
fx.ptr_store(nope[c4], rawb + (64 * c4 + 16 * g) // 4)
for cc in range_constexpr(ROPE // 32):
fx.ptr_store(rope[cc], rawb + (NOPE + (cc * 32 + 8 * g) * 2) // 4)
if g == 0:
fx.ptr_store(scw, rawb + DATA // 4)
gpu.barrier()
stamp(c, 5)
# QK: A = the keys (dequantized once), B = q of a head tile (LDS); the
# contraction's 32-dim chunk 2 c + h of the nope dims is {64 c + 16 g +
# 8 h ..} (lane g's 16 B load), q read in the same order
scale = fx.Int32(a["qk_scale"]).bitcast(fx.Float32)
st = sl["st"]
accs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(nht)]
for c4 in range_constexpr(NOPE // 64):
ksc = pow2(_scale_byte(scw, c4))
for h2 in range_constexpr(2):
kw = fp8x8_bf16(nope[c4][2 * h2], nope[c4][2 * h2 + 1], ksc).bitcast(
fx.BFloat16
)
for ht in range_constexpr(nht):
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
qw = fx.Vector(
fx.ptr_load(
qb + 32 * c4 + 8 * g + 4 * h2,
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
for cc in range_constexpr(ROPE // 32):
kw = rope[cc].bitcast(fx.BFloat16)
for ht in range_constexpr(nht):
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
qw = fx.Vector(
fx.ptr_load(
qb + (NOPE + cc * 32 + 8 * g) // 2,
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
# scores of keys 4 g + i (C rows), head r16 of a tile (C column)
ok = (key < kv_len) & okv
okw = ok.select(fx.Int32(1), fx.Int32(0))
valid = [lane_gather(okw, 4 * g + i) != 0 for i in range(4)]
svs = []
for ht in range_constexpr(nht):
sv = [
fx.Float32(valid[i].select(accs[ht][i] * scale, fx.Float32(NEG)))
for i in range(4)
]
mw = butterfly(
fx.max(fx.max(sv[0], sv[1]), fx.max(sv[2], sv[3])), (32, 16), fx.max
)
if g == 0:
fx.ptr_store(mw, st + ((hgi * nht + ht) * nkt + kt) * HEAD_TILE + r16)
svs.append(sv)
gpu.barrier()
lst = st + d.heads * nkt
for ht in range_constexpr(nht):
hti = hgi * nht + ht
m = fx.ptr_load(st + (hti * nkt) * HEAD_TILE + r16)
for j in range_constexpr(1, nkt):
m = fx.max(m, fx.ptr_load(st + (hti * nkt + j) * HEAD_TILE + r16))
sv = svs[ht]
p = [
fx.Float32(valid[i].select(hw_exp2(sv[i] - m), fx.Float32(0.0)))
for i in range(4)
]
lw = (p[0] + p[2]) + (p[1] + p[3])
lw = lw + xshfl(lw, 32)
lw = lw + xshfl(lw, 16)
pw = (
fx.Vector.from_elements(p, fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)
)
pbase = (
sl["p"] + (hti * HEAD_TILE + r16) * p_row(d, s) + (kt * 16 + 4 * g) // 2
)
fx.ptr_store(fx.Vector.from_elements([pw[0], pw[1]], fx.Int32), pbase)
if g == 0:
fx.ptr_store(lw, lst + (hti * nkt + kt) * HEAD_TILE + r16)
gpu.barrier()
# chunk stats: (max, sum) per head
if tid < d.heads:
hts = tid // HEAD_TILE
hr = tid % HEAD_TILE
mm = fx.ptr_load(st + (hts * nkt) * HEAD_TILE + hr)
ll = fx.ptr_load(lst + (hts * nkt) * HEAD_TILE + hr)
for j in range_constexpr(1, nkt):
mm = fx.max(mm, fx.ptr_load(st + (hts * nkt + j) * HEAD_TILE + hr))
ll = ll + fx.ptr_load(lst + (hts * nkt + j) * HEAD_TILE + hr)
bo.buffer_store(
fx.Vector.from_elements([mm, ll], fx.Float32),
rsrc(c["pstat"]),
((t * nchunk + ch) * d.heads + tid) * 2,
cache_modifier=CM_DEV,
)
# PV: wave w dims 64 w + 4 r16 + dt (dt = 0..3: one LDS word holds the
# four of a key); the rope wave (dims 448..) reads bf16 pairs. Lane
# (r16, g) takes keys 4 g + j of a 16-key tile: A[dim][key]
is_rope = wave == WAVES - 1
dbase = 64 * wave
sh = (wave % 4) * 8
accs = [
[fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(4)]
for _ in range(d.head_tiles)
]
for kt2 in range_constexpr(nkt):
rb0 = sl["raw"] + (kt2 * 16 + 4 * g) * (REC // 4)
vv = fx.Vector.filled(16, 0.0, fx.Float32) # [dt][j]
if is_rope:
rv = []
for j in range_constexpr(4):
rw = fx.Vector(
fx.ptr_load(
rb0 + j * (REC // 4) + (NOPE + 8 * r16) // 4,
result_type=fx.Vector.make_type(2, fx.Int32),
)
)
rv.append([bf_lo(rw[0]), bf_hi(rw[0]), bf_lo(rw[1]), bf_hi(rw[1])])
vv = fx.Vector.from_elements(
[rv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
)
else:
nv = []
for j in range_constexpr(4):
rb = rb0 + j * (REC // 4)
w = fx.Int32(fx.ptr_load(rb + (dbase + 4 * r16) // 4))
scb = fx.Int32(fx.ptr_load(rb + DATA // 4 + wave // 4))
scl = pow2((scb >> sh) & 0xFF)
nv.append(
_fp8_pair_f32(w, scl, False) + _fp8_pair_f32(w, scl, True)
)
vv = fx.Vector.from_elements(
[nv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
)
for ht2 in range_constexpr(d.head_tiles):
pb = fx.Vector(
fx.ptr_load(
sl["p"]
+ (ht2 * HEAD_TILE + r16) * p_row(d, s)
+ (kt2 * 16 + 4 * g) // 2,
result_type=fx.Vector.make_type(2, fx.Int32),
)
).bitcast(fx.Int16)
for dt in range_constexpr(4):
av = (
fx.Vector.from_elements(
[vv[dt * 4 + j] for j in range(4)], fx.Float32
)
.to(fx.BFloat16)
.bitcast(fx.Int16)
)
accs[ht2][dt] = fx.Vector(
rocdl.mfma_f32_16x16x16bf16_1k(
T.vec(4, T.f32), [av, pb, accs[ht2][dt]]
)
)
stamp(c, 6)
# partial[t][ch][head][dim] bf16: lane (r16, g) holds C rows 4 g + i ->
# dims 64 w + 16 g + 4 i + dt of head ht2 * 16 + r16: 32 contiguous
# bytes, two 16 B stores (4 lanes fill a 128 B line)
for ht2 in range_constexpr(d.head_tiles):
hh = ht2 * HEAD_TILE + r16
words = []
for i in range_constexpr(4):
v4 = fx.Vector.from_elements(
[accs[ht2][dt][i] for dt in range(4)], fx.Float32
)
w2 = v4.to(fx.BFloat16).bitcast(fx.Int32)
words += [w2[0], w2[1]]
off = (((t * nchunk + ch) * d.heads + hh) * HEAD_DIM + dbase + 16 * g) // 2
for hlf in range_constexpr(2):
bo.buffer_store(
fx.Vector.from_elements(words[4 * hlf : 4 * hlf + 4], fx.Int32),
rsrc(c["part"]),
off + 4 * hlf,
cache_modifier=CM_DEV,
)
publish(c, c["srdy"], t * nchunk + ch)
stamp(c, 7)
# ---------------------------------------------------------------- combine
COMB_HEADS = WAVES # heads a combine unit: a wave each
@traced
def stage_combine(c, unit):
"""(token, 8 heads) ``unit``, a wave a head, 8 dims a lane: the chunks
merged with the sink folded in, bf16, inverse RoPE, MXFP8 of each 32 -> XO,
published (a flag a (token, head group))."""
lane, wave, d, s = c["lane"], c["wave"], c["d"], c["S"]
a = c["args"]
hg_n = d.heads // COMB_HEADS
t = unit // hg_n
hg = unit % hg_n
h = hg * COMB_HEADS + wave
nchunk = c["nchunk"]
ck = chunk_keys(d, s)
kv_len = ld_i32(a["klen"], t)
nlive = (kv_len + ck - 1) // ck
pos = ld_i32(a["pos"], 2 * t)
sink = ld_f32(a["sink"], h) * fx.Float32(LOG2E)
d0 = lane * 8 # this lane's 8 dims
rope_lane = d0 >= NOPE
jj = fx.max((d0 - NOPE) // 2, 0) # 4 pairs from here
cs = [
rope_lane.select(ld_f32(a["cos_sin"], pos * ROPE + jj + i), fx.Float32(1.0))
for i in range(4)
]
sn = [
rope_lane.select(
ld_f32(a["cos_sin"], pos * ROPE + HALF + jj + i), fx.Float32(0.0)
)
for i in range(4)
]
await_flags(c, c["srdy"], nlive, lambda u: t * nchunk + u)
m = fx.Float32(NEG)
ms, ls, vs = [], [], []
for cc in range_constexpr(nchunk):
live = cc < nlive
stv = fx.Vector(
bo.buffer_load(
rsrc(c["pstat"]),
((t * nchunk + cc) * d.heads + h) * 2,
vec_width=2,
dtype=T.f32,
)
)
pv = fx.Vector(
bo.buffer_load(
rsrc(c["part"]),
(((t * nchunk + cc) * d.heads + h) * HEAD_DIM + d0) // 2,
vec_width=4,
dtype=T.i32,
)
)
mc = fx.Float32(live.select(stv[0], fx.Float32(NEG)))
ms.append(mc)
ls.append(fx.Float32(live.select(stv[1], fx.Float32(0.0))))
vals = []
for e in range_constexpr(4):
vals += [bf_lo(pv[e]), bf_hi(pv[e])]
vs.append([fx.Float32(live.select(v, fx.Float32(0.0))) for v in vals])
m = fx.max(m, mc)
m_final = fx.max(m, sink)
lsum = hw_exp2(sink - m_final)
tot = [fx.Float32(0.0)] * 8
for cc in range_constexpr(nchunk):
wgt = hw_exp2(ms[cc] - m_final)
lsum = lsum + ls[cc] * wgt
tot = [tot[e] + vs[cc][e] * wgt for e in range(8)]
inv = fx.Float32(1.0) / lsum
o = [bf16_round(tot[e] * inv) for e in range(8)]
# inverse GPT-J RoPE of each pair: e' = e cos + o sin, o' = o cos - e sin
y = []
for i in range_constexpr(4):
e, od = o[2 * i], o[2 * i + 1]
y += [e * cs[i] + od * sn[i], od * cs[i] - e * sn[i]]
amax = abs(y[0])
for v in y[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max) # 32 dims = 4 lanes
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(y[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(y[4 + i] * mul) for i in range(4)])
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["xo"]),
t * (d.q_rows // 4) + (h * HEAD_DIM + d0) // 4,
cache_modifier=CM_DEV,
)
if lane % 4 == 0:
bo.buffer_store(
code,
rsrc(c["xos"]),
t * (d.q_rows // 32) + (h * HEAD_DIM + d0) // 32,
cache_modifier=CM_DEV,
)
publish(c, c["crdy"], t * hg_n + hg)
# ---------------------------------------------------------------- wo_a / wo_b
@traced
def stage_woa(c, task, ops, t0, n):
"""Rows 32 task .. of wo_a -> bf16 -> vLLM's MXFP8 of the 32 -> X8B."""
tid, red, d, lane = c["tid"], c["red"], c["d"], c["lane"]
split = WOA_SPLIT_K
gemv_mfmas(c, d.group_k, c["xl"], c["xsl"], red, ops, split, tiled=True, rows=n)
gpu.barrier()
mine = tid < WOA_ROWS * n
tl = fx.min(tid // WOA_ROWS, n - 1)
t = plus(t0, tl)
row = tid % WOA_ROWS
tile = row // ROWS
y = bf16_round(
row_sum(red, row % ROWS, tl, [tile * split + w for w in range(split)])
)
amax = butterfly(abs(y), (1, 2, 4, 8, 16), fx.max)
code = mx_code(amax)
q = clamp_fp8(y * mx_mul(code))
w = fp8_pack4(
q,
*[
lane_gather(q.bitcast(fx.Int32), fx.min(lane + k, 63)).bitcast(fx.Float32)
for k in (1, 2, 3)
],
)
if mine & (row % 4 == 0):
bo.buffer_store(
w,
rsrc(c["x8b"]),
t * (d.o_rows // 4) + (task * WOA_ROWS + row) // 4,
cache_modifier=CM_DEV,
)
if mine & (row == 0):
bo.buffer_store(
code, rsrc(c["x8bs"]), t * (d.o_rows // 32) + task, cache_modifier=CM_DEV
)
gpu.barrier()
def woa_loads(c, task):
a = c["args"]
return gemv_loads(
c,
a["woa"],
a["woa_s"],
c["d"].group_k,
task * WOA_ROWS,
WOA_SPLIT_K,
tiled=True,
)
def wob_loads(c, task, pred=None):
a = c["args"]
return gemv_loads(c, a["wob"], a["wob_s"], c["d"].o_rows, task * ROWS, pred=pred)
def _wob_store(a, t, col, v0, v1):
word = (
fx.Vector.from_elements([v0, v1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
bo.buffer_store(word, rsrc(a["out"]), (t * a["out_stride"] + col) // 2)
@traced
def stage_wob(c, task, ops, task2, ops2, has2, t0, n, mark=False):
"""Rows 16 task .. (and 16 task2 .., ``has2``) of wo_b -> bf16 -> out, or
(``c["wob_out"]``, the mono layer) the caller's hook with each bf16 pair."""
tid, red = c["tid"], c["red"]
o_rows = c["d"].o_rows
a = c["args"]
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red, ops, rows=n)
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red + RED, ops2, rows=n)
gpu.barrier()
if const_expr(mark):
stamp(c, 15)
per = ROWS * n // 2 # bf16 pairs a task
k = tid // per
emit = c.get("wob_out") or (lambda t, col, v0, v1: _wob_store(a, t, col, v0, v1))
if (tid < 2 * per) & ((k == 0) | has2):
u = tid % per
rp = u % (ROWS // 2)
tl = u // (ROWS // 2)
t = plus(t0, tl)
rk = red + k * RED
col = (k == 0).select(task, task2) * ROWS + 2 * rp
v0 = row_sum(rk, 2 * rp, tl)
v1 = row_sum(rk, 2 * rp + 1, tl)
emit(t, col, v0, v1)
gpu.barrier()
# ---------------------------------------------------------------- kernel
def back_plan(key):
"""(nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob) of a build."""
s, d = key.tokens, Dims(key.tp)
ck = chunk_keys(d, s)
nchunk = cdiv(KEYS, ck)
n_split = s * nchunk
n_comb = s * (d.heads // COMB_HEADS)
COMB0 = n_split % BLOCKS
assert d.woa_tasks <= BLOCKS
assert BLOCKS < d.wob_tasks <= 2 * BLOCKS
# the GEMVs past the attention units, so CTAs without attention work have
# their weights in flight from the start
WOA0 = (n_split + n_comb) % BLOCKS
WOB0 = (WOA0 + d.woa_tasks) % BLOCKS
# wo_b's weights before the wo_a stage only while the wo_a CTAs have no
# attention work (loads complete in issue order: else they would hold up
# wo_a's activation rows)
early_wob = n_split + n_comb + d.woa_tasks <= BLOCKS
return nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob
def back_lds_members(s: int, d: Dims) -> dict:
"""K2's attention LDS structs: the split stage's and the GEMVs' (a union's
members: one at a time)."""
rows = tile_rows(s)
ck = chunk_keys(d, s)
@fx.struct
class SplitLds:
q: fx.Array[fx.Int32, d.heads * Q_ROW, 16]
raw: fx.Array[fx.Int32, ck * REC // 4, 16]
p: fx.Array[fx.Int32, d.heads * p_row(d, s), 16]
st: fx.Array[fx.Float32, 2 * d.heads * (WAVES // head_groups(d, s)), 16]
@fx.struct
class GemvLds:
red: fx.Array[fx.Float32, 2 * RED, 16]
xl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 4), 16]
xsl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 32), 16]
return {"split": SplitLds, "gemv": GemvLds}
def back_lds(s: int, d: Dims):
"""K2's attention LDS: the union of ``back_lds_members``."""
m = back_lds_members(s, d)
SplitLds, GemvLds = m["split"], m["gemv"]
@fx.union
class BackLds:
split: SplitLds # type: ignore[valid-type]
gemv: GemvLds # type: ignore[valid-type]
return BackLds
def back_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
"""The attention back's context: LDS views (``lds``: an allocated
``back_lds``, or a dict of the members' views), mailbox (tag ``tag``),
scratch regions, arguments by name."""
s, d = key.tokens, Dims(key.tp)
tid = fx.thread_idx.x
if isinstance(lds, dict):
spl, gml = lds["split"], lds["gemv"]
else:
spl, gml = lds.split.peek(), lds.gemv.peek()
c = {
"S": s,
"tid": tid,
"bid": fx.block_idx.x,
"lane": tid % 64,
"wave": tid // 64,
"red": gml.red.ptr,
"xl": gml.xl.ptr,
"xsl": gml.xsl.ptr,
"d": d,
"ratio": key.ratio,
"mb": Mailbox(tag),
"nchunk": back_plan(key)[0],
"sl": {"q": spl.q.ptr, "raw": spl.raw.ptr, "p": spl.p.ptr, "st": spl.st.ptr},
"args": args,
}
for region, (off, _) in (
layout or back_scratch(s, d, start=front_end(s, d))
).items():
c[region] = scratch + fx.Int64(off)
return c
EPOCH_MARKS = 4 # word offset of the per-CTA marks in the epoch buffer
@traced
def epoch_begin(c, epoch, ep):
"""This CTA has read the launch pair's epoch: its own mark word (no
contended atomic) := ep + 1, which ``epoch_end`` checks."""
if c["tid"] == 0:
bo.buffer_store(
ep + 1, rsrc(epoch), EPOCH_MARKS + c["bid"], cache_modifier=CM_DEV
)
@traced
def epoch_end(c, epoch, ep, reset=None):
"""CTA 0, once every CTA marked this epoch (thread u checks CTA u's mark):
``reset()`` (state later pairs must find cleared), then the epoch moved on.
The next pair's hand-offs carry a tag no earlier pair wrote; later launches'
kernels start after this one ends."""
if c["bid"] == 0:
tid = c["tid"]
if tid < BLOCKS:
r = rsrc(epoch)
def load_mark():
return fx.Int32(
bo.buffer_load(
r,
EPOCH_MARKS + tid,
vec_width=1,
dtype=T.i32,
cache_modifier=CM_DEV,
)
)
_ = spin_until(load_mark, lambda v: v != ep + 1)
gpu.barrier()
if tid == 0:
if const_expr(reset is not None):
reset()
gstore(epoch, ep + 1, words=1)
@traced
def run_back(c, key, bid):
"""The attention back's stages on this CTA: split, combine, wo_a, wo_b."""
s = key.tokens
dd = Dims(key.tp)
# plain ints: a Python object read inside traced control flow is carried
woa_tasks, groups, heads, group_k = dd.woa_tasks, dd.groups, dd.heads, dd.group_k
q_rows, o_rows, wob_tasks = dd.q_rows, dd.o_rows, dd.wob_tasks
a = c["args"]
tiles = token_tiles(s)
_, n_split, n_comb, COMB0, WOA0, WOB0, early_wob = back_plan(key)
stamp(c, 0)
for unit in range(first_task(bid, 0), n_split, BLOCKS):
stage_split(c, unit)
stamp(c, 1)
for unit in range(first_task(bid, COMB0), n_comb, BLOCKS):
stage_combine(c, unit)
gpu.barrier()
stamp(c, 2)
# the GEMV weights, every CTA right after its attention work: its wo_a
# task (if any), then its one or two wo_b tasks
woa_task = first_task(bid, WOA0)
has_woa = woa_task < woa_tasks
woa_t = fx.min(woa_task, woa_tasks - 1)
ops = gemv_loads(
c,
a["woa"],
a["woa_s"],
group_k,
woa_t * WOA_ROWS,
WOA_SPLIT_K,
tiled=True,
pred=has_woa,
)
wob_task = first_task(bid, WOB0)
wob_second = wob_task + BLOCKS
has_second = wob_second < wob_tasks
def wob_weights():
return (
wob_loads(c, wob_task),
wob_loads(c, fx.min(wob_second, wob_tasks - 1), pred=has_second),
)
if const_expr(early_wob):
ops1, ops2 = wob_weights()
if const_expr(c.get("tl") is not None):
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
stamp(c, 8)
# wo_a: every token tile of its group's XO rows, the next tile's in flight
if has_woa:
grp = woa_t // (woa_tasks // groups)
await_flags(c, c["crdy"], s, lambda u: u * (heads // COMB_HEADS) + grp)
stamp(c, 9)
xo = (
c["xo"],
c["xos"],
q_rows // 4,
grp * group_k // 4,
group_k // 4,
q_rows // 32,
grp * group_k // 32,
group_k // 32,
)
regs = rows_loads(c, *xo, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, group_k // 4, group_k // 32, c["xl"], c["xsl"], n)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *xo, *tiles[ti + 1])
stage_woa(c, woa_t, ops, t0, n)
stamp(c, 10)
publish(c, c["ardy"], woa_t)
stamp(c, 3)
# wo_b
if const_expr(not early_wob):
ops1, ops2 = wob_weights()
stamp(c, 11)
await_flags(c, c["ardy"], woa_tasks, lambda u: u)
stamp(c, 12)
x8b = (
c["x8b"],
c["x8bs"],
o_rows // 4,
0,
o_rows // 4,
o_rows // 32,
0,
o_rows // 32,
)
regs = rows_loads(c, *x8b, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, o_rows // 4, o_rows // 32, c["xl"], c["xsl"], n)
if const_expr(ti == 0):
stamp(c, 14)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *x8b, *tiles[ti + 1])
stage_wob(
c,
wob_task,
ops1,
fx.min(wob_second, wob_tasks - 1),
ops2,
has_second,
t0,
n,
mark=ti == 0,
)
if const_expr(ti == 0):
stamp(c, 13)
stamp(c, 4)
@@ -0,0 +1,504 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Device primitives of the DeepSeek-V4.1 attention mega kernel.
Wave reductions and shuffles, hardware math, bf16 / fp8 / MX packing, the
scaled MFMA, buffer and 64-bit global accesses, and the tagged-pair mailbox.
Adapted from ATOM's mono decode framework (``atom/mono/device``), itself ported
from the FlyDSL GLM-5 layer kernel.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
from aiter.ops.flydsl.kernels.kernels_common import kernel_signature
from flydsl._mlir import ir
from flydsl._mlir.dialects import llvm
from flydsl.compiler.ast_rewriter import ASTRewriter
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T, as_ir_value
# The execution model: one CTA a CU of an MI355X, all resident.
BLOCKS = 256
THREADS = 512
WAVES = THREADS // 64
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16, NT = 2. SC1 alone is
# device scope (past the per-XCD L2s); NT streams data read once.
CM_DEV = 16
CM_NT = 2
FP8_MAX = 448.0
FP8, FP4 = 0, 4 # f8f6f4 MFMA operand formats
UNIT_SCALE = 127 # E8M0 code of 1.0
FLT_MIN = 1.1754943508222875e-38
POLL_MAX = 12 # mailbox pairs polled a batch
def traced(fn):
"""FlyDSL's AST rewrite for a helper, so its ``if`` / ``while`` / ``for`` on
traced values lower to scf ops (kernels rewrite only their own body)."""
return ASTRewriter.transform(fn)
def kernel_symbol(stem, **params):
"""Kernel symbol that profilers demangle to ``aiter::<stem>_<params>``."""
name = f"{stem}_{kernel_signature(**params)}"
return f"_ZN5aiter{len(name)}{name}E"
def first_task(bid, base):
"""CTA ``bid``'s first task of a stage placed from CTA ``base``, then every
BLOCKS-th."""
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
# ----------------------------------------------------------------------------- memory
def rsrc(addr, nbytes=None):
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
def ld_i32(ptr, i, cm=0):
return fx.Int32(
bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.i32, cache_modifier=cm)
)
def ld_f32(ptr, i):
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
def ld_bf(ptr, i):
return fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
)
def ld_i64(ptr, i):
"""Element i of an i64 buffer (two dwords)."""
w = fx.Vector(bo.buffer_load(rsrc(ptr), i * 2, vec_width=2, dtype=T.i32))
return (fx.Int64(w[1]) << 32) | fx.Int64(fx.Uint32(w[0]))
def _gptr(addr):
return llvm.IntToPtrOp(
ir.Type.parse("!llvm.ptr<1>"), as_ir_value(fx.Int64(addr))
).result
def gload(addr, words=4, nt=False):
"""``words`` i32 (1, 2 or 4) at a 64-bit global address: a cache past 4 GiB is
reachable only this way (a buffer offset is 32 bits)."""
i32 = ir.IntegerType.get_signless(32)
ty = i32 if words == 1 else ir.VectorType.get([words], i32)
v = llvm.LoadOp(ty, _gptr(addr), alignment=4 * words, nontemporal=nt or None).result
return fx.Int32(v) if words == 1 else fx.Vector(v)
def atomic_add_dev(addr, value):
"""Device-scope atomic add on the global i32 at ``addr``; the old value.
Relaxed: an acquire / release here would write back and invalidate L2."""
return fx.Int32(
fx.atomic_add(
_gptr(addr),
fx.Int32(value),
syncscope="agent",
ordering=fx.AtomicOrdering.Monotonic,
)
)
def gstore(addr, value, words=4):
"""Store ``words`` i32 (an Int32 or a Vector) at a 64-bit global address."""
v = as_ir_value(fx.Int32(value) if words == 1 else value)
llvm.StoreOp(v, _gptr(addr), alignment=4 * words)
# ----------------------------------------------------------------------------- math
def fresh(v):
"""``v`` through an opaque move, so the lane math built on it is computed at
its use instead of being hoisted out of a task loop and held in registers."""
return fx.Int32(
llvm.inline_asm(
T.i32,
[fx.Int32(v).ir_value()],
"v_mov_b32 $0, $1",
"=v,v",
has_side_effects=True,
)
)
def tile_map(tid, width):
"""(tid // width, tid % width) for 0 <= tid < THREADS without a division:
shifts for a power of two, else a multiply-shift exact on that range."""
if width & (width - 1) == 0:
sh = width.bit_length() - 1
return tid >> sh, tid & (width - 1)
m = -(-(1 << 20) // width)
assert all((t * m) >> 20 == t // width for t in range(THREADS)), width
row = (tid * m) >> 20
return row, tid - row * width
def uniform(v):
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
def _hw_f32(name, x):
return fx.Float32(
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
)
def hw_rsq(x):
return _hw_f32("llvm.amdgcn.rsq.f32", x)
def hw_rcp(x):
return _hw_f32("llvm.amdgcn.rcp.f32", x)
def hw_exp2(x):
return _hw_f32("llvm.amdgcn.exp2.f32", x)
def fma(a, b, c):
return fx.Float32(fmath.fma(fx.Float32(a), fx.Float32(b), fx.Float32(c)))
def div_rn(x, d, r):
"""``x / d`` rounded to nearest (IEEE), given ``r = RN(1 / d)``."""
q = x * r
return fma(fma(-q, d, x), r, q)
def permlane_swap(off, x, y):
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
pr = swap(
llvm.StructType.get_literal([T.i32, T.i32]),
as_ir_value(fx.Int32(x)),
as_ir_value(fx.Int32(y)),
False,
False,
)
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
def lane_gather(v, src_lane):
"""Lane ``src_lane``'s value of v (ds_bpermute)."""
return fx.Int32(
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
)
def lane_f32(v, src_lane):
return lane_gather(v.bitcast(fx.Int32), src_lane).bitcast(fx.Float32)
def readlane(v, src_lane):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
r = fx.Int32(rocdl.readlane(T.i32, bits, fx.Int32(src_lane)))
return r.bitcast(fx.Float32) if isinstance(v, fx.Float32) else r
def xshfl(v, off):
"""Value of lane ``lane ^ off`` (DPP in a row, permlane swaps across)."""
assert off in (1, 2, 4, 8, 16, 32), off
if off >= 16:
return v.shuffle_xor(off, 64)
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
if off == 8:
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
elif off == 4:
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
elif off == 2:
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
else:
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
return y.bitcast(fx.Float32) if is_f else y
def xred(v, off, op):
if off < 16:
return op(v, xshfl(v, off))
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
a, b = permlane_swap(off, x, x)
if is_f:
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
return op(type(v)(a), type(v)(b))
def _add(a, b):
return a + b
def butterfly(v, offsets, op=_add):
"""Butterfly reduction over the xor ``offsets``, in that order."""
for off in offsets:
v = xred(v, off, op)
return v
def wave_sum(v):
return butterfly(v, (32, 16, 8, 4, 2, 1))
def wave_max(v):
return butterfly(v, (32, 16, 8, 4, 2, 1), fx.max)
def row_shr(v, lane, k):
"""Lane - k of the same 16-lane row, else 0 (DPP row_shr)."""
moved = lane_gather(v.bitcast(fx.Int32), fx.max(lane - k, 0)).bitcast(fx.Float32)
return fx.Float32((lane % 16 >= k).select(moved, fx.Float32(0.0)))
# ----------------------------------------------------------------------------- packing
def bf16_round(a):
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
def bf16_pair(a, b):
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
return (
fx.Vector.from_elements([a, b], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Float32)[0]
)
def bf_lo(w):
return (w << 16).bitcast(fx.Float32)
def bf_hi(w):
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
def fp8_pack4(a, b, c, d):
"""Four f32 (in range) -> one dword of four E4M3 bytes, RNE."""
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
def fp8x8_bf16(w0, w1, scale):
"""Eight E4M3 of two dwords times an f32 ``scale`` -> eight bf16 as a
4-dword Vector (exact for a power-of-two scale in range)."""
out = []
for w in (w0, w1):
for hi in (False, True):
p = rocdl.cvt_scalef32_pk_bf16_fp8(
T.bf16x2, as_ir_value(fx.Int32(w)), as_ir_value(fx.Float32(scale)), hi
)
out.append(fx.Vector(p).bitcast(fx.Int32)[0])
return fx.Vector.from_elements(out, fx.Int32)
def clamp_fp8(v):
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
def pow2(code):
"""The fp32 power of two a biased exponent (1..254) stands for."""
return (code << 23).bitcast(fx.Float32)
def rcp_pow2(code):
"""1 / ``pow2(code)``, exact for code in [1, 253]."""
return ((254 - code) << 23).bitcast(fx.Float32)
def ceil_exp(v):
"""Biased exponent of the smallest power of two >= v (v > 0 finite, normal)."""
bits = v.bitcast(fx.Int32)
return ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
fx.Int32(1), fx.Int32(0)
)
def mx_code(amax):
"""VLLM's ROCm MXFP8 scale code of a 32-group: clamp(ceil(log2(amax / 448))
+ 127, 0, 254), amax floored at FLT_MIN."""
a = fx.max(amax, fx.Float32(FLT_MIN))
q = div_rn(a, fx.Float32(FP8_MAX), fx.Float32(1.0 / FP8_MAX))
return fx.min(fx.max(ceil_exp(q), fx.Int32(0)), fx.Int32(254))
def mx_mul(code):
"""2^(127 - code) as fp32, for code in [0, 254] (the quant multiplier)."""
return ((254 - code) << 23).bitcast(fx.Float32)
# ----------------------------------------------------------------------------- MFMA
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8):
"""16x16x128 scaled MFMA (gfx950 f8f6f4): lane l (row l % 16) holds K bytes
16 (l // 16) .. +16 and 64 + 16 (l // 16) .. +16 of the 128-step, and the
E8M0 of K block l // 16."""
return fx.Vector(
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, 0, sa, 0, sb]
)
)
def mfma_bf16(a, b, c):
"""16x16x32 bf16 MFMA: lane l (row / col l % 16) holds K 8 (l // 16) .. +8."""
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
# ----------------------------------------------------------------------------- mailbox
@traced
def _spin(load_all, pending):
v = load_all()
while pending(v):
rocdl.s_sleep(1)
v = load_all()
return v
def spin_until(load, pending):
"""``load()`` again (s_sleep between) while ``pending(value)``; the value."""
return _spin(load, pending)
class Mailbox:
"""Tagged-pair hand-off between the CTAs of one launch (device scope).
Every 32-bit value is stored next to the launch's tag in one 8-byte store; a
consumer polls until every tag matches, so a hand-off is one round trip with
no separate flag. The tag is ``epoch * 64 + layer + 1`` (``epoch`` a device
counter bumped once a step): a pair from an earlier layer or step never
carries a live tag, and nothing is cleared between launches."""
def __init__(self, tag):
self.tag = tag
def put(self, base, i, v):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
bo.buffer_store(
fx.Vector.from_elements([bits, self.tag], fx.Int32),
rsrc(base),
i * 2,
cache_modifier=CM_DEV,
)
def put_bf(self, base, i, vs):
"""Elements i .. i + len(vs) (2 or 4, i aligned) as bf16 pairs, one store."""
words = []
for j in range_constexpr(len(vs) // 2):
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(base),
i,
cache_modifier=CM_DEV,
)
def put_words(self, base, i, words):
"""Raw 32-bit words (1 or 2) at pairs i.. (fp8 words must come this way)."""
vals = []
for w in words:
vals += [fx.Int32(w), self.tag]
bo.buffer_store(
fx.Vector.from_elements(vals, fx.Int32),
rsrc(base),
i * 2,
cache_modifier=CM_DEV,
)
def poll(self, specs, batch=POLL_MAX):
"""Batched poll of ``specs`` = [(base, pair index, npairs in {1, 2})]: all
re-loaded while any tag is stale. Returns the value words."""
if const_expr(len(specs) == 0):
return []
if const_expr(len(specs) > batch):
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
tag = self.tag
def load_all():
words = []
for b, i, n in specs:
w = fx.Vector(
bo.buffer_load(
rsrc(b),
fx.Int32(i) * 2,
vec_width=2 * n,
dtype=T.i32,
cache_modifier=CM_DEV,
)
)
words += [w[e] for e in range(2 * n)]
return fx.Vector.from_elements(words, fx.Int32)
nw = sum(2 * n for _, _, n in specs)
def pending(v):
bad = v[1] != tag
for e in range_constexpr(3, nw, 2):
bad = bad | (v[e] != tag)
return bad
v = _spin(load_all, pending)
outs, e = [], 0
for _, _, n in specs:
outs.append([v[e + 2 * q] for q in range(n)])
e += 2 * n
return outs
LOAD_BATCH = 4
@traced
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
"""A CTA's strided loop over v = tid + THREADS i < n: ``batch`` rounds' loads
issued before their uses; a round past n loads element n - 1, uses nothing."""
rounds = -(-n // THREADS)
for b0 in range_constexpr(0, rounds, batch):
idx = range(b0, min(b0 + batch, rounds))
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
for i, g in zip(idx, got):
v = tid + THREADS * i
if v < n:
use(v, g)
def barrier():
gpu.barrier()
# ----------------------------------------------------------------------------- timeline
def memrealtime():
"""s_memrealtime: the 100 MHz device clock."""
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
@traced
def stamp(c, point):
"""A timeline build's stamp: thread 0 of the CTA stores the clock at
``point`` (``c["tl"]`` [BLOCKS, points] i64; None: not a timeline build)."""
if const_expr(c.get("tl") is None):
return
if c["tid"] == 0:
t = memrealtime()
w = fx.Vector.from_elements([fx.Int32(t), fx.Int32(t >> 32)], fx.Int32)
bo.buffer_store(w, rsrc(c["tl"]), (c["bid"] * c["tl_points"] + point) * 2)
@@ -0,0 +1,680 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""K1 ``front``: one DeepSeek-V4.1 attention layer from its normed input row to
the rotated query and the step's KV rows in the sliding-window cache.
One launch a layer and rank, ``BLOCKS`` CTAs x ``THREADS`` threads, S rows.
Every CTA issues its weight loads (its wqkv part and its wq_b tasks) at the
kernel start, before it waits on anything; the stages:
xq (S > 16: S x 2 K halves, on the 32 CTAs without a wqkv part): the
bf16 input's half -> vLLM's MXFP8 -> X8 (published, a flag each)
kt (S): the token's attention keys for K2 -- top-k compressed slots
through the kv source's block table, then window slots -> KT
wqkv_a (112 row tiles x 2 K halves, 16 rows): the input's MXFP8 half --
quantized in the CTA (S <= 16) or X8's -- a token tile at a time
(the next tile's rows in flight) times wqkv's -> fp32 partials ->
QKVP
qkv (S): the halves summed -> bf16; q_norm -> bf16 -> MXFP8 -> QX8
(published, a flag a token); kv_norm -> bf16 -> GPT-J RoPE -> bf16
-> the fp8_ds_mla record at the token's slot
wq_b (H x 512 / 16, 16 rows): QX8 times wq_b -> bf16 -> GPT-J RoPE of
each head's dims 448.. -> q (global, bf16)
Each elementwise step follows vLLM's ROCm path (``mxfp8_quantize``,
``FusedQKVRMSNorm``, ``fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert``);
the GEMV reductions differ from the Triton GEMMs' only in their order.
Adapted from ATOM's V4.1 ``attn_pre`` (K1), less its mHC seam.
"""
from dataclasses import dataclass
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T
from .device import (
BLOCKS,
CM_DEV,
THREADS,
Mailbox,
bf16_round,
bf_hi,
bf_lo,
butterfly,
ceil_exp,
clamp_fp8,
div_rn,
first_task,
fp8_pack4,
fresh,
gstore,
hw_rsq,
ld_f32,
ld_i32,
mx_code,
mx_mul,
rsrc,
stamp,
tile_map,
traced,
xshfl,
)
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
from .plan import (
DATA,
EPS,
HALF,
HEAD_DIM,
HIDDEN,
KEYS,
KV_DIM,
NOPE,
Q_RANK,
QKV_ROWS,
ROPE,
ROWS,
TILE,
TOPK,
WINDOW,
WQKV_KPARTS,
WQKV_TASKS,
Dims,
front_scratch,
tile_rows,
token_tiles,
)
_STREAM = fx.Stream(None)
X_WORDS = HIDDEN // 4
X_GROUPS = HIDDEN // 32
QX_WORDS = Q_RANK // 4
KSTEPS = HIDDEN // 128 // WQKV_KPARTS # a wqkv part's 128-steps
KWORDS = KSTEPS * 32
KGROUPS = KSTEPS * 4
XH = HIDDEN // WQKV_KPARTS # a K half's values
XH_COLS = XH // 8 // 2 # 16 B items (8 values) a half of a K half
XH_RPR = THREADS // XH_COLS # half-rows a round
XQ_ITEMS = XH // 8 # an xq unit's 16 B items (one a thread)
FRONT_POINTS = 6 # start, kt done, wqkv done, qkv done, wq_b done, (spare)
@dataclass(frozen=True)
class FrontBuild:
tokens: int
tp: int
ratio: int # the layer's compress ratio (0: no top-k keys)
timeline: bool = False
def plus(t0, v):
return v if isinstance(t0, int) and t0 == 0 else v + t0
@traced
def publish(c, region, i):
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if c["tid"] == 0:
c["mb"].put(region, i, fx.Int32(1))
@traced
def await_flags(c, region, n, index):
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
tid = c["tid"]
if tid < n:
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
gpu.barrier()
# ---------------------------------------------------------------- xq
def xq_loads(c, unit, pred):
"""(token, K half) ``unit``'s bf16 input: its load only, 8 values a thread
(threads past the half mirror its last item)."""
a = c["args"]
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
col = fx.min(c["tid"], XQ_ITEMS - 1)
e = (unit >> 1) * a["x_stride"] + (unit & 1) * XH + col * 8
return fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32))
@traced
def stage_xq(c, unit, w, live):
"""``xq_loads``' values -> vLLM's MXFP8 -> X8 words / X8S codes of the
unit (published by the caller)."""
tid, lane = c["tid"], c["lane"]
vals = []
for d in range_constexpr(4):
vals += [bf_lo(w[d]), bf_hi(w[d])]
amax = abs(vals[0])
for v in vals[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max)
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
t = unit >> 1
part = unit & 1
if live & (tid < XQ_ITEMS):
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["x8"]),
t * X_WORDS + part * KWORDS + tid * 2,
cache_modifier=CM_DEV,
)
if lane % 4 == 0:
bo.buffer_store(
code,
rsrc(c["x8s"]),
t * X_GROUPS + part * KGROUPS + (tid >> 2),
cache_modifier=CM_DEV,
)
# ---------------------------------------------------------------- kt
@traced
def stage_kt(c, t):
"""Token t's keys for K2: ``KT[t][k] = slot | 1 << 31`` for its top-k compressed
rows (k < ntopk), the window slot for k in [ntopk, kv_len), -1 for an
invalid or absent key; KLEN[t] = kv_len (0 for a pad row), KLEN[S + t] =
ntopk."""
tid = c["tid"]
a, ratio = c["args"], c["ratio"]
# device scope where this launch reads the table (K2's)
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
slot = ld_i32(a["slot"], 2 * t)
live = slot >= 0
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
if const_expr(ratio > 0):
pos = ld_i32(a["pos"], 2 * t)
ntopk = live.select(fx.min((pos + 1) // ratio, fx.Int32(TOPK)), fx.Int32(0))
req = ld_i32(a["t2r"], t)
else:
ntopk = fx.Int32(0)
kv_len = ntopk + nswa
for i in range_constexpr((KEYS + THREADS - 1) // THREADS):
k = fresh(tid) + THREADS * i
sidx = fx.min(fx.max(k - ntopk, 0), WINDOW - 1)
s_slot = ld_i32(a["swa_idx"], t * WINDOW + sidx)
val = (k < kv_len).select(s_slot, fx.Int32(-1))
if const_expr(ratio > 0):
local = ld_i32(a["topk"], t * TOPK + fx.min(k, TOPK - 1))
lc = fx.max(local, 0)
e = a["comp_block"]
blk = ld_i32(a["comp_bt"], req * a["bt_stride"] + lc // e)
c_slot = (blk * e + lc % e) | fx.Int32(-2147483648)
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
val = (k < ntopk).select(c_val, val)
if k < KEYS:
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
if tid == 0:
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
# ---------------------------------------------------------------- wqkv_a
def x_half_loads(c, part, pred, t0, n):
"""Tokens t0 .. t0 + n's bf16 input, K half ``part``: its loads only (16 B a
lane-item, 8 values; a thread one column of [2 n half-rows, 160]), masked
off on CTAs without a wqkv part."""
a = c["args"]
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
hr0, col = tile_map(c["tid"], XH_COLS)
e0 = part * XH + col * 8
out = []
for i in range((2 * n + XH_RPR - 1) // XH_RPR):
hr = fx.min(hr0 + XH_RPR * i, 2 * n - 1)
e = (t0 + (hr >> 1)) * a["x_stride"] + (hr & 1) * (XH // 2) + e0
out.append(fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32)))
return out
@traced
def quant_x_half(c, regs, n):
"""``x_half_loads``' values -> vLLM's MXFP8 rows of the K half in LDS (a
32-group = 4 lanes)."""
tid, lane = c["tid"], c["lane"]
x_row, s_row = lds_row(KWORDS), lds_row(KGROUPS)
hr0, col = tile_map(tid, XH_COLS)
act = tid < XH_RPR * XH_COLS
for i in range_constexpr(len(regs)):
hr = hr0 + XH_RPR * i
w = regs[i]
vals = []
for d in range_constexpr(4):
vals += [bf_lo(w[d]), bf_hi(w[d])]
amax = abs(vals[0])
for v in vals[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max)
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
tok = hr >> 1
half = hr & 1
if act & (hr < 2 * n):
fx.ptr_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
c["xl"] + (tok * x_row + half * (KWORDS // 2) + col * 2),
)
if lane % 4 == 0:
fx.ptr_store(
code, c["xsl"] + (tok * s_row + half * (KGROUPS // 2) + (col >> 2))
)
gpu.barrier()
@traced
def stage_wqkv(c, task, part, ops, t0, n, live):
"""Rows 16 task .. of wqkv over K half ``part`` against tokens t0 .. t0 + n
-> fp32 partials QKVP[part] (``live``: a CTA with a wqkv part)."""
tid, red = c["tid"], c["red"]
gemv_mfmas(c, HIDDEN, c["xl"], c["xsl"], red, ops, rows=n, nsteps=KSTEPS)
gpu.barrier()
if (tid < ROWS * n) & live:
r = tid % ROWS
tl = tid // ROWS
t = plus(t0, tl)
v = row_sum(red, r, tl)
c["mb"].put(c["qkvp"], (part * c["S"] + t) * QKV_ROWS + task * ROWS + r, v)
gpu.barrier()
# ---------------------------------------------------------------- q / kv norms
@traced
def stage_qkv(c, t):
"""Token t: the K halves summed -> bf16; q_norm -> bf16 -> MXFP8 (QX8);
kv_norm -> bf16 -> RoPE -> bf16 -> the fp8_ds_mla record at
slot_mapping[t]. 256 threads (8 q columns and 2 kv columns each); threads
256.. mirror thread 255 and store nothing."""
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
a, s = c["args"], c["S"]
tt = fx.min(tid, 255)
mine = tid < 256
qc = [fx.min(tt * 8 + j, Q_RANK - 1) for j in range(8)]
# independent loads first: they overlap the partials' wait
qw = [c["ldbf"](a["qn_w"], qc[j]) for j in range(8)]
kw = [c["ldbf"](a["kvn_w"], tt * 2 + j) for j in range(2)]
pos = ld_i32(a["pos"], 2 * t)
slot = ld_i32(a["slot"], 2 * t)
jr = tt - NOPE // 2
is_rope = jr >= 0
jj = fx.max(jr, 0)
cs = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + jj), fx.Float32(1.0))
sn = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + HALF + jj), fx.Float32(0.0))
specs = []
for p in range_constexpr(2):
specs += [(c["qkvp"], (p * s + t) * QKV_ROWS + qc[j], 1) for j in range(8)]
specs += [
(c["qkvp"], (p * s + t) * QKV_ROWS + Q_RANK + tt * 2 + j, 1)
for j in range(2)
]
got = c["mb"].poll(specs)
vals = [
bf16_round(got[j][0].bitcast(fx.Float32) + got[10 + j][0].bitcast(fx.Float32))
for j in range(10)
]
xq = [((tt * 8 + j) < Q_RANK).select(vals[j], fx.Float32(0.0)) for j in range(8)]
xk = vals[8:10]
ssq = xq[0] * xq[0]
for j in range_constexpr(1, 8):
ssq = ssq + xq[j] * xq[j]
ssq = butterfly(ssq, (32, 16, 8, 4, 2, 1))
ssk = butterfly(xk[0] * xk[0] + xk[1] * xk[1], (32, 16, 8, 4, 2, 1))
if (lane == 0) & mine:
fx.ptr_store(ssq, red + wave)
fx.ptr_store(ssk, red + 4 + wave)
gpu.barrier()
tq = (fx.ptr_load(red + 0) + fx.ptr_load(red + 1)) + (
fx.ptr_load(red + 2) + fx.ptr_load(red + 3)
)
tk = (fx.ptr_load(red + 4) + fx.ptr_load(red + 5)) + (
fx.ptr_load(red + 6) + fx.ptr_load(red + 7)
)
rq = hw_rsq(
div_rn(tq, fx.Float32(float(Q_RANK)), fx.Float32(1.0 / Q_RANK))
+ fx.Float32(EPS)
)
rk = hw_rsq(
div_rn(tk, fx.Float32(float(KV_DIM)), fx.Float32(1.0 / KV_DIM))
+ fx.Float32(EPS)
)
# q: bf16 normed row, then vLLM's MXFP8 (a 32-group = 4 threads)
ys = [bf16_round((xq[j] * rq) * qw[j]) for j in range(8)]
amax = abs(ys[0])
for y in ys[1:]:
amax = fx.max(amax, abs(y))
code = mx_code(butterfly(amax, (1, 2), fx.max))
mul = mx_mul(code)
if const_expr(c.get("index", False)): # noqa: SIM102
# the bf16 qr, vLLM's indexer's input
if mine & (tt * 8 < Q_RANK):
words = [
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
for i in range(4)
]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a["qr_out"]),
t * (Q_RANK // 2) + 4 * tt,
)
if mine & (tt * 8 < Q_RANK):
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["qx8"]),
t * QX_WORDS + 2 * tt,
cache_modifier=CM_DEV,
)
if tt % 4 == 0:
bo.buffer_store(
code,
rsrc(c["qx8s"]),
t * (Q_RANK // 32) + tt // 4,
cache_modifier=CM_DEV,
)
# kv: this thread's pair (2 tt, 2 tt + 1)
kn = [bf16_round((xk[j] * rk) * kw[j]) for j in range(2)]
e0 = bf16_round(kn[0] * cs - kn[1] * sn)
e1 = bf16_round(kn[0] * sn + kn[1] * cs)
# record quant of the nope dims: a 64-block = 32 threads (half a wave)
amax = butterfly(fx.max(abs(e0), abs(e1)), (1, 2, 4, 8, 16), fx.max)
rcode = ceil_exp(
div_rn(
fx.max(amax, fx.Float32(1e-4)), fx.Float32(448.0), fx.Float32(1.0 / 448.0)
)
)
inv = mx_mul(rcode)
q0 = clamp_fp8(e0 * inv)
q1 = clamp_fp8(e1 * inv)
n0 = xshfl(q0, 1)
n1 = xshfl(q1, 1)
word = fp8_pack4(q0, q1, n0, n1)
rope_word = (
fx.Vector.from_elements([e0, e1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
blk = a["swa_block"]
sblock = slot // blk
soff = slot % blk
base = a["swa"] + fx.Int64(sblock) * fx.Int64(a["swa_stride"])
rec = base + fx.Int64(soff * DATA)
if mine & (tt % 32 == 0) & (tt < NOPE // 2):
fx.ptr_store(rcode.bitcast(fx.Float32), red + 8 + tt // 32)
gpu.barrier()
if mine & (slot >= 0):
if tt < NOPE // 2:
if tt % 2 == 0:
gstore(rec + fx.Int64(2 * tt), word, words=1)
else:
gstore(rec + fx.Int64(NOPE + 4 * (tt - NOPE // 2)), rope_word, words=1)
if tt == 0:
sb = [fx.ptr_load(red + 8 + b).bitcast(fx.Int32) for b in range(7)]
w0 = sb[0] | (sb[1] << 8) | (sb[2] << 16) | (sb[3] << 24)
w1 = sb[4] | (sb[5] << 8) | (sb[6] << 16)
sc = base + fx.Int64(blk * DATA + soff * 8)
gstore(sc, fx.Vector.from_elements([w0, w1], fx.Int32), words=2)
publish(c, c["qrdy"], t)
# ---------------------------------------------------------------- wq_b
def wqb_rope(c, task0, pos_t, n):
"""(cos, sin) of this thread's pair in ``stage_wqb``'s epilogue over a tile
of n rows (``pos_t``: its token's position; every task of the CTA shares
the pair's dims: tasks BLOCKS apart are whole heads apart)."""
a = c["args"]
rp = c["tid"] % (ROWS // 2)
dim = (task0 * ROWS) % HEAD_DIM + 2 * rp
jj = fx.max((dim - NOPE) >> 1, 0)
return ld_f32(a["cos_sin"], pos_t * ROPE + jj), ld_f32(
a["cos_sin"], pos_t * ROPE + HALF + jj
)
def wqb_pos(c, t0, n):
"""This thread's token position for ``stage_wqb``'s epilogue of tile
t0 .. t0 + n."""
per = ROWS * n // 2
tl = fx.min((c["tid"] % per) // (ROWS // 2), n - 1)
return ld_i32(c["args"]["pos"], 2 * (t0 + tl))
@traced
def stage_wqb(c, task0, bops, t0, n, rope):
"""Rows 16 (task0 + BLOCKS k) .. of the rank's H x 512 query rows, every
task k of the CTA: GEMVs -> bf16 -> GPT-J RoPE of a head's dims 448..
(fp32) -> bf16 -> q."""
tid, red, d = c["tid"], c["red"], c["d"]
a = c["args"]
nt = len(bops)
for k in range_constexpr(nt):
gemv_mfmas(c, Q_RANK, c["qxl"], c["qxsl"], red + k * RED, bops[k], rows=n)
gpu.barrier()
per = ROWS * n // 2 # bf16 pairs a task
if tid < nt * per:
k = tid // per
u = tid % per
rp = u % (ROWS // 2)
tl = u // (ROWS // 2)
t = plus(t0, tl)
r0 = (task0 + BLOCKS * k) * ROWS + 2 * rp
rk = red + k * RED
v0 = bf16_round(row_sum(rk, 2 * rp, tl))
v1 = bf16_round(row_sum(rk, 2 * rp + 1, tl))
is_rope = (task0 * ROWS) % HEAD_DIM + 2 * rp >= NOPE
cs, sn = rope
e0 = is_rope.select(v0 * cs - v1 * sn, v0)
e1 = is_rope.select(v0 * sn + v1 * cs, v1)
word = (
fx.Vector.from_elements([e0, e1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
bo.buffer_store(word, rsrc(a["q"]), (t * d.q_rows + r0) // 2)
gpu.barrier()
# ---------------------------------------------------------------- kernel
def front_lds(s: int, d: Dims):
"""K1's LDS (the attention front's stages)."""
rows = tile_rows(s)
wqb_per_cta = d.wqb_tasks // BLOCKS
@fx.struct
class FrontLds:
red: fx.Array[fx.Float32, wqb_per_cta * RED + 16, 16]
qxl: fx.Array[fx.Int32, rows * lds_row(QX_WORDS), 16]
qxsl: fx.Array[fx.Int32, rows * lds_row(Q_RANK // 32), 16]
xl: fx.Array[fx.Int32, rows * lds_row(KWORDS), 16]
xsl: fx.Array[fx.Int32, rows * lds_row(KGROUPS), 16]
return FrontLds
def front_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
"""The front stages' context: LDS views, mailbox (tag ``tag``), scratch
regions, arguments (``args``: the kernel's, by name)."""
s, d = key.tokens, Dims(key.tp)
tid = fx.thread_idx.x
c = {
"S": s,
"tid": tid,
"bid": fx.block_idx.x,
"lane": tid % 64,
"wave": tid // 64,
"red": lds.red.ptr,
"xl": lds.xl.ptr,
"xsl": lds.xsl.ptr,
"qxl": lds.qxl.ptr,
"qxsl": lds.qxsl.ptr,
"mb": Mailbox(tag),
"d": d,
"ratio": key.ratio,
"ldbf": lambda ptr, i: fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
),
"args": args,
}
for region, (off, _) in (layout or front_scratch(s, d)).items():
c[region] = scratch + fx.Int64(off)
return c
@traced
def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
"""The front's stages on this CTA. ``x8_given``: a stage before this one
(the mono layer's seam norm) publishes X8 / XRDY for every token (no input
quantization here). ``before_wqkv()`` runs ahead of the wqkv weights' loads
(a latency-critical stage of the caller's), ``after_wqkv()`` right after."""
s = key.tokens
wqb_tasks = Dims(key.tp).wqb_tasks
a = c["args"]
tid = c["tid"]
tiles = token_tiles(s)
wqb_per_cta = wqb_tasks // BLOCKS
assert wqb_tasks % BLOCKS == 0
n_wqkv = WQKV_TASKS * WQKV_KPARTS # 224
assert n_wqkv <= BLOCKS
n_free = BLOCKS - n_wqkv # CTAs without a wqkv part: xq, kt
QKV0 = 0 # on wqkv CTAs
WQB0 = 0
in_cta_quant = s <= TILE and not x8_given
xq_stage = s > TILE and not x8_given
xq_units = WQKV_KPARTS * s
xq_per = (xq_units + n_free - 1) // n_free
stamp(c, 0)
# ---- Loads complete in issue order, so every CTA issues its latency-
# critical loads before its later stages' weights: a wqkv CTA its first
# input tile and its wqkv part, the next tiles while it quantizes, then
# its wq_b tasks; a CTA with a key-table task those after that work.
# The q / kv norms run on wqkv CTAs, their partials' wait behind the
# wq_b weights that are due by then anyway.
wqkv_unit = first_task(bid, 0)
has_wqkv = wqkv_unit < n_wqkv
task = fx.min(wqkv_unit, n_wqkv - 1) % WQKV_TASKS
part = fx.min(wqkv_unit, n_wqkv - 1) // WQKV_TASKS
wqb_task = first_task(bid, WQB0)
free = bid - n_wqkv
is_free = free >= 0
if const_expr(in_cta_quant):
xregs = [x_half_loads(c, part, has_wqkv, *tiles[0])]
if const_expr(xq_stage):
xunits = [free + n_free * i for i in range(xq_per)]
xlive = [is_free & (u < xq_units) for u in xunits]
xq_regs = [
xq_loads(c, fx.min(fx.max(u, 0), xq_units - 1), lv)
for u, lv in zip(xunits, xlive)
]
if const_expr(before_wqkv is not None):
before_wqkv()
kops = gemv_loads(
c,
a["wqkv"],
a["wqkv_s"],
HIDDEN,
task * ROWS,
kbeg=part * KSTEPS,
nsteps=KSTEPS,
pred=has_wqkv,
)
if const_expr(after_wqkv is not None):
after_wqkv()
if const_expr(xq_stage): # noqa: SIM102 (a build-time and a run-time test)
if is_free:
for i in range_constexpr(xq_per):
stage_xq(
c, fx.min(fx.max(xunits[i], 0), xq_units - 1), xq_regs[i], xlive[i]
)
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if tid < xq_per:
u = free + n_free * tid
if u < xq_units:
c["mb"].put(c["xrdy"], u, fx.Int32(1))
if const_expr(not c.get("index", False)):
# an index source's top-k comes after this launch: K2 builds its table
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
stage_kt(c, t)
stamp(c, 1)
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
# ---- wqkv_a: one (row tile, K half) a CTA, every token tile; the wq_b
# weights issued behind the input rows' loads
def wqb_weights():
return [
gemv_loads(c, a["wqb"], a["wqb_s"], Q_RANK, (wqb_task + BLOCKS * i) * ROWS)
for i in range(wqb_per_cta)
]
if const_expr(in_cta_quant):
if has_wqkv:
quant_x_half(c, xregs[0], tiles[0][1])
bops = wqb_weights()
if has_wqkv:
stage_wqkv(c, task, part, kops, *tiles[0], has_wqkv)
else:
await_flags(c, c["xrdy"], s, lambda u: WQKV_KPARTS * u + part)
xr = (
c["x8"],
c["x8s"],
X_WORDS,
part * KWORDS,
KWORDS,
X_GROUPS,
part * KGROUPS,
KGROUPS,
)
regs = rows_loads(c, *xr, tiles[0][0], tiles[0][1], pred=has_wqkv)
if const_expr(len(tiles) == 1):
bops = wqb_weights()
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, KWORDS, KGROUPS, c["xl"], c["xsl"], n)
if const_expr(ti + 1 < len(tiles)):
t1, n1 = tiles[ti + 1]
regs = rows_loads(c, *xr, t1, n1, pred=has_wqkv)
if const_expr(ti + 2 == len(tiles)):
bops = wqb_weights()
stage_wqkv(c, task, part, kops, t0, n, has_wqkv)
stamp(c, 2)
# ---- q / kv norms, KV insert
for t in range(first_task(bid, QKV0), s, BLOCKS):
stage_qkv(c, t)
stamp(c, 3)
# ---- wq_b: every task's rows of a token tile at a time, the next
# tile's rows in flight
ropes = [wqb_rope(c, wqb_task, poss[ti], tiles[ti][1]) for ti in range(len(tiles))]
await_flags(c, c["qrdy"], s, lambda u: u)
qx = (c["qx8"], c["qx8s"], QX_WORDS, 0, QX_WORDS, Q_RANK // 32, 0, Q_RANK // 32)
regs = rows_loads(c, *qx, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, QX_WORDS, Q_RANK // 32, c["qxl"], c["qxsl"], n)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *qx, *tiles[ti + 1])
stage_wqb(c, wqb_task, bops, t0, n, ropes[ti])
stamp(c, 4)
@@ -0,0 +1,234 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 GEMV of the mega kernel: 16 rows of a row-major e4m3 weight [N, K]
(vLLM's loaded layout, E8M0 scales in 32x32 blocks [N / 32, K / 32]) against
every token of a tile, its MXFP8 rows in LDS, on the gfx950 scaled MFMA.
A task may cover a window of K (``kbeg`` .. ``kbeg + nsteps`` 128-steps): a
split-K part, its activations in LDS holding just that window. The weight
operands are loaded first (all in flight, before the task waits on its
activations); wave w takes the window's steps w, w + 8, ... (or, ``split``, a
contiguous range of waves 0 .. split - 1), its 16 x 16 partial in ``red``,
summed in wave order by ``row_sum``. Adapted from ATOM's
``attn_pre.gemv_fp8_*``, which reads aiter's (16, 16)-preshuffled layout.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr
from flydsl.expr.typing import T
from .device import THREADS, WAVES, mfma_scaled, rsrc, tile_map, traced
# a token's LDS row of MXFP8 words / E8M0 codes, padded by 16 B so 16 rows read
# by one MFMA operand do not share a bank group
LDS_PAD = 4
RED = WAVES * 64 * 4 # one task's accumulators in ``red`` (f32)
def lds_row(words):
return words + LDS_PAD
def lds_at(u, row):
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
return u + u // row * LDS_PAD
def _steps(c, nsteps, split, tiled):
"""This wave's steps of the window: [(local step, live)]."""
wave = c["wave"]
if const_expr(split is None):
per = (nsteps + WAVES - 1) // WAVES
ragged = nsteps % WAVES != 0
return [
(wave + WAVES * i, (wave + WAVES * i < nsteps) if ragged else None)
for i in range(per)
]
assert split <= WAVES and nsteps % split == 0
per = nsteps // split
part = wave % split if tiled else fx.min(wave, split - 1)
return [(part * per + i, None) for i in range(per)]
def _step_load(c, r_w, r_ws, k, row0, st, pred=None):
"""Global step ``st``'s weight operand of rows row0 .. +16: two 16 B chunks a
lane (K bytes 16 j .. and 64 + 16 j .. of the step, row row0 + lane % 16)
and the E8M0 byte of its K block. ``pred`` false: masked (out of range: no
memory traffic, zeros)."""
lane = c["lane"]
j = lane // 16
row = row0 + lane % 16
base = row * k + 128 * st + 16 * j
lo = fx.Vector(bo.buffer_load(r_w, base // 4, vec_width=4, dtype=T.i32))
hi = fx.Vector(bo.buffer_load(r_w, (base + 64) // 4, vec_width=4, dtype=T.i32))
kg = k // 32
wsi = (row0 // 32) * kg + st * 4 + j
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
def gemv_loads(
c, w, ws, k, row0, split=None, tiled=False, kbeg=0, nsteps=None, pred=None
):
"""The weight operands of rows row0 .. +16 (``tiled``: row0 + 16 (wave //
split)) over steps kbeg .. kbeg + nsteps for this wave, all in flight
(``pred`` false: masked off, no memory traffic)."""
nsteps = k // 128 if nsteps is None else nsteps
if const_expr(pred is None):
r_w, r_ws = rsrc(w), rsrc(ws)
else:
# a masked-off CTA reads through a zero-size buffer: zeros, no traffic
nb = pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
r_w, r_ws = rsrc(w, nb), rsrc(ws, nb)
if const_expr(tiled):
row0 = row0 + 16 * (c["wave"] // split)
return [
_step_load(
c,
r_w,
r_ws,
k,
row0,
kbeg + (st if live is None else fx.min(st, nsteps - 1)),
pred,
)
for st, live in _steps(c, nsteps, split, tiled)
]
def _step_mfma(c, nsteps, xl, xsl, st, ops, acc, live, rows):
lane = c["lane"]
lo, hi, sa = ops
x_row, s_row = lds_row(nsteps * 32), lds_row(nsteps * 4)
j = lane // 16
col = fx.min(lane % 16, rows - 1)
if const_expr(live is not None):
st = fx.min(st, nsteps - 1)
av = fx.Vector.from_elements(
[lo[d] for d in range(4)] + [hi[d] for d in range(4)], fx.Int32
)
xlo = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xhi = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + 16 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xv = fx.Vector.from_elements(
[xlo[d] for d in range(4)] + [xhi[d] for d in range(4)], fx.Int32
)
sb = fx.ptr_load(xsl + (col * s_row + st * 4 + j))
if const_expr(live is not None):
# a step past the end adds zero at unit scale
av = fx.Vector.from_elements(
[live.select(av[d], fx.Int32(0)) for d in range(8)], fx.Int32
)
sa = live.select(sa, fx.Int32(127))
sb = live.select(sb, fx.Int32(127))
return mfma_scaled(av, xv, acc, sa, sb)
def gemv_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=16, nsteps=None):
"""``gemv_loads``' operands against ``rows`` tokens' MXFP8 rows of the
window in LDS (``xl`` words, ``xsl`` codes, rows ``lds_row`` of the window)
-> ``red[wave][lane]`` (a 16 x 16 C: row 4 (lane // 16) + i, token lane % 16)."""
lane, wave = c["lane"], c["wave"]
nsteps = k // 128 if nsteps is None else nsteps
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i, (st, live) in enumerate(_steps(c, nsteps, split, tiled)):
acc = _step_mfma(c, nsteps, xl, xsl, st, ops[i], acc, live, rows)
if const_expr(split is not None and not tiled):
acc = fx.Vector.from_elements(
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
fx.Float32,
)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
def row_sum(red, r, t, waves=None):
"""Row r, token t of the waves' accumulators, summed in wave order."""
tot = fx.Float32(0.0)
for w in waves if waves is not None else range(WAVES):
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
return tot
def span_items(width, n):
"""A [n, width] span of 16 B items: rows a round of THREADS threads, rounds."""
wq = width // 4
rpr = THREADS // wq
assert rpr >= 1, width
return wq, rpr, (n + rpr - 1) // rpr
def rows_loads(
c,
words_base,
codes_base,
stride_w,
off_w,
words,
stride_g,
off_g,
groups,
t0,
n,
pred=None,
):
"""Tokens t0 .. t0 + n's published MXFP8 row slices -- ``words`` i32 from
``off_w`` and ``groups`` codes (an i32 each) from ``off_g``, rows
``stride_w`` / ``stride_g`` i32 apart -- loaded (issued, not waited on),
16 B a lane-item: a thread one column, rows a round apart. Plain loads:
their flags are seen, and nothing in the launch read these lines before.
``pred`` false: masked off (zeros, no memory traffic)."""
tid = c["tid"]
nb = None if pred is None else pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
out = []
for base, stride, off, width in (
(words_base, stride_w, off_w, words),
(codes_base, stride_g, off_g, groups),
):
wq, rpr, rounds = span_items(width, n)
row0, col = tile_map(tid, wq)
a0 = col * 4 + off + t0 * stride
r = rsrc(base, nb)
regs = []
for i in range(rounds):
row = row0 + rpr * i
if (i + 1) * rpr > n or rpr * wq < THREADS:
row = fx.min(row, n - 1)
regs.append(
fx.Vector(
bo.buffer_load(r, row * stride + a0, vec_width=4, dtype=T.i32)
)
)
out.append(regs)
return out
@traced
def _store_span(c, regs, width, dst, n):
tid = c["tid"]
wq, rpr, _ = span_items(width, n)
row0, col = tile_map(tid, wq)
lrow = lds_row(width)
base = dst + (row0 * lrow + col * 4)
for i in range_constexpr(len(regs)):
if const_expr((i + 1) * rpr <= n and rpr * wq == THREADS):
fx.ptr_store(regs[i], base + i * rpr * lrow)
else:
if (row0 + rpr * i < n) & (tid < rpr * wq):
fx.ptr_store(regs[i], base + i * rpr * lrow)
def rows_store(c, regs, words, groups, xl, xsl, n):
"""``rows_loads``' registers into LDS rows 0 .. n (``lds_row`` strides)."""
_store_span(c, regs[0], words, xl, n)
_store_span(c, regs[1], groups, xsl, n)
gpu.barrier()
@@ -0,0 +1,173 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Geometry of the DeepSeek-V4.1 attention mega kernel: model widths, a TP
rank's share of them, task counts and the scratch (mailbox) layout."""
from dataclasses import dataclass
HIDDEN = 5120
Q_RANK = 1280
KV_DIM = 512
QKV_ROWS = Q_RANK + KV_DIM # 1792
HEAD_DIM = 512
NOPE = 448
ROPE = 64
HALF = ROPE // 2
N_HEADS = 64
O_GROUPS = 8
O_RANK = 1024
WINDOW = 128
TOPK = 512
EPS = 1e-20
ROWS = 16 # GEMV rows a task
TILE = 16 # tokens a GEMV pass (an MFMA's N)
HEAD_TILE = 16 # heads an attention MFMA takes
WOA_ROWS = 32 # a wo_a task's rows: one MX group of its output
MAX_TOKENS = 48
# KV record (V4 fp8_ds_mla): 576 B data rows (448 e4m3 | 64 bf16), then 8 B of
# scales a token (7 UE8M0 + pad) after the block's rows
DATA = 576
SCALE_BYTES = 8
# attention: a split is 16 keys; a token has at most TOPK + WINDOW keys
BK = 16
SPLITS = 64 # split slots a (token, head tile): 1024 keys >= TOPK + WINDOW
COLS = 32 # PV columns a group (one MX group of the output)
GROUPS = HEAD_DIM // COLS # 16
PAIR = 8 # a mailbox (value, tag) pair
def cdiv(a, b):
return -(-a // b)
def round_up(a, b):
return cdiv(a, b) * b
@dataclass(frozen=True)
class Dims:
tp: int
def __post_init__(self):
assert self.tp in (2, 4), self.tp
@property
def heads(self) -> int:
return N_HEADS // self.tp
@property
def head_tiles(self) -> int:
return self.heads // HEAD_TILE
@property
def groups(self) -> int:
return O_GROUPS // self.tp
@property
def group_k(self) -> int:
"""wo_a's K a group: its eight heads' outputs."""
return self.heads // self.groups * HEAD_DIM # 4096
@property
def o_rows(self) -> int:
return self.groups * O_RANK
@property
def q_rows(self) -> int:
return self.heads * HEAD_DIM
@property
def wqb_tasks(self) -> int:
return self.q_rows // ROWS
@property
def woa_tasks(self) -> int:
return self.o_rows // WOA_ROWS
@property
def wob_tasks(self) -> int:
return HIDDEN // ROWS # 320
WQKV_TASKS = QKV_ROWS // ROWS # 112
WQKV_KPARTS = 2 # wqkv's K split: 224 (row tile, K half) parts
KEYS = TOPK + WINDOW # a token's attention keys at most
def token_tiles(s):
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
def tile_rows(s):
return min(s, TILE)
def pair_layout(items, start=0, align=256):
out, off = {}, start
for name, pairs in items:
off = round_up(off, align)
out[name] = (off, pairs * PAIR)
off += pairs * PAIR
return out
def byte_layout(items, start=0, align=256):
out, off = {}, start
for name, nbytes in items:
off = round_up(off, align)
out[name] = (off, nbytes)
off += nbytes
return out
def front_scratch(s: int, d: Dims) -> dict:
"""K1's scratch regions -> (byte offset, bytes): published data (plain
words behind a flag) and mailbox pairs."""
return byte_layout(
[
("x8", s * HIDDEN), # e4m3 words (S > 16: quantized once, shared)
("x8s", s * HIDDEN // 32 * 4), # an i32 code a group
("xrdy", WQKV_KPARTS * s * PAIR), # a flag a (token, K half)
("qkvp", WQKV_KPARTS * s * QKV_ROWS * PAIR),
("qx8", s * Q_RANK), # e4m3 words
("qx8s", s * Q_RANK // 32 * 4), # an i32 code a group
("qrdy", s * PAIR),
]
)
def back_scratch(s: int, d: Dims, start: int = 0) -> dict:
"""K2's scratch regions, laid out past ``start``: published data (plain)
and flag pairs."""
ck = (
64 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= 256 else 128
) # back.chunk_keys
nchunk = cdiv(KEYS, ck)
return byte_layout(
[
("part", s * nchunk * d.heads * HEAD_DIM * 2), # bf16 partials
("pstat", s * nchunk * d.heads * 8), # (max, sum) f32
("srdy", s * nchunk * PAIR),
("xo", s * d.q_rows), # e4m3 words
("xos", s * d.q_rows // 32 * 4), # an i32 code a group
("crdy", s * d.heads // 8 * PAIR),
("x8b", s * d.o_rows),
("x8bs", s * d.o_rows // 32 * 4),
("ardy", d.woa_tasks * PAIR),
],
start=start,
)
def front_end(s: int, d: Dims) -> int:
"""Where K2's regions start: past K1's, which share the launch tag."""
return round_up(max(o + n for o, n in front_scratch(s, d).values()), 256)
def scratch_bytes(s: int, d: Dims) -> int:
b = back_scratch(s, d, start=front_end(s, d))
return round_up(max(o + n for o, n in b.values()), 4096)
@@ -0,0 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/__init__.py
"""The mono kernels' shared mechanisms: ``ops`` (wave / math primitives, the TP
group's peer buffers), ``sync`` (the tagged mailbox), ``mx`` (FP8 / MX
arithmetic), ``plan`` (execution model, layouts, build keys) and
``peer_memory`` (symmetric peer buffers)."""
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/mx.py
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
tile's operand loads and the scaled MFMA that takes them.
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
FP8_MAX = 448.0
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
FP8, FP4 = 0, 4
UNIT_SCALE = 127 # the E8M0 code of 1.0
def clamp_fp8(v):
"""``v`` clamped to the E4M3 range."""
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
def code_ceil(v):
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
clamped to [1, 254]: the codes a scale may take."""
bits = v.bitcast(fx.Int32)
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
fx.Int32(1), fx.Int32(0)
)
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
def rcp_pow2(code):
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
bit for bit, a multiply where the division is a dozen instructions."""
return ((254 - code) << 23).bitcast(fx.Float32)
def scale_index(row, col, cols):
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
r32, a, b = row // 32, (row // 16) % 2, row % 16
c8, d, e = col // 8, (col // 4) % 2, col % 4
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
def fp4_tile_load(r_w, base, rg, k, st, at):
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
Nontemporal: a weight a step reads once."""
return fx.Vector(
bo.buffer_load(
r_w,
base + rg * (2 * k) + st * 256 + at,
vec_width=4,
dtype=T.i32,
cache_modifier=CM_NT,
)
)
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
byte ``sb_byte``)."""
return fx.Vector(
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
)
)
@@ -0,0 +1,300 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/ops.py
# atom/mono/device/ranks.py
# atom/mono/device/stamps.py
"""Device-side primitives of the mono kernels: wave reductions and shuffles, the
approximate hardware math, bf16 / fp8 packing, buffer resources, the
execution-model constants and the kernel symbol naming.
Ported from the FlyDSL GLM-5 layer kernel (ROCm/FlyDSL branch
``glm5-mla-moe-layer-monokernel`` @ 1e28fd8a,
``kernels/mla_moe_layer/shared_reuse_moe_kernel.py``). Only published flydsl APIs
and aiter's vendored ``buffer_ops`` / ``dpp_utils`` are used.
Every function but ``kernel_symbol`` is traced inside a ``@flyc.kernel`` body.
Raw boundaries kept, for want of an equivalent on the pinned FlyDSL:
``buffer_ops`` loads / stores (the kernels take raw Int64 addresses, no tensors
for ``make_buffer_tensor``); raw MFMA intrinsics (hand-laid operands: permlane
transposes, fp8 packing, MXFP4 scales); the LLVM intrinsics here (``expect``,
``s_memrealtime``, the approximate rsq / rcp / exp2) and the struct unpacking of
``permlane*_swap``.
Also the TP group seen from inside a kernel (every rank's peer buffer, an
all-reduce's rank-ordered sum) and a timeline build's phase stamps."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
from flydsl._mlir.dialects import llvm
from flydsl.compiler.ast_rewriter import ASTRewriter
from flydsl.expr import const_expr, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T, as_ir_value
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES
# ---------------------------------------------------------------- primitives
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16. SC1 alone is device
# scope (past the per-XCD caches); SC0 | SC1 is system scope (peers over XGMI).
CM_DEV = 16
CM_SYS = 17
# NT = 2: nontemporal. A weight a layer reads once streams past the caches it
# would only evict the step's reused data from (the MoE at 48 rows: 143 -> 133 us)
CM_NT = 2
def rsrc(addr, nbytes=None):
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
def permlane_swap(off, x, y):
"""v_permlane{32,16}_swap on registers (x, y): off 32 trades x's upper-half
lanes with y's lower-half lanes; off 16 trades x's odd 16-lane rows with y's
even rows (each within its half). -> (x', y')."""
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
pr = swap(
llvm.StructType.get_literal([T.i32, T.i32]),
as_ir_value(fx.Int32(x)),
as_ir_value(fx.Int32(y)),
False,
False,
)
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
def lane_gather(v, src_lane):
"""Lane ``src_lane``'s value of v, a source lane a lane (ds_bpermute)."""
return fx.Int32(
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
)
def memrealtime():
"""s_memrealtime: the 100 MHz device clock (no FlyDSL wrapper)."""
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
def fresh(v):
"""``v`` (a per-lane Int32) through an opaque move, here: the lane math built
on it is computed at its use, where LLVM would otherwise hoist it out of a task
loop or CSE it across stages into registers held for the whole kernel (spills,
and a kernel with scratch is refused: ``runtime.widths``)."""
return fx.Int32(
llvm.inline_asm(
T.i32,
[fx.Int32(v).ir_value()],
"v_mov_b32 $0, $1",
"=v,v",
has_side_effects=True,
)
)
def uniform(v):
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
def _hw_f32(name, x):
return fx.Float32(
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
)
def hw_rsq(x):
"""v_rsq_f32 (~1 ulp, no libm range fixups)."""
return _hw_f32("llvm.amdgcn.rsq.f32", x)
def hw_rcp(x):
return _hw_f32("llvm.amdgcn.rcp.f32", x)
def hw_exp2(x):
return _hw_f32("llvm.amdgcn.exp2.f32", x)
def div_rn(x, d, r):
"""``x / d`` rounded to nearest, bit-identical to the IEEE division, given
``r = 1.0 / d`` from one true division per divisor (Markstein: a quotient
within 1 ulp plus its exact fma residual times RN(1/d) rounds correctly)."""
q = x * r
return fx.Float32(fmath.fma(fx.Float32(fmath.fma(-q, d, x)), r, q))
def xshfl(v, off):
"""Value of lane ``lane ^ off``: offsets 32 / 16 via v_permlane*_swap, in-row
offsets via DPP (VALU latency, not LDS). Powers of two only: any other
offset would take the lane ^ 1 path."""
assert off in (1, 2, 4, 8, 16, 32), off
if off >= 16:
return v.shuffle_xor(off, 64)
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
if off == 8:
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
elif off == 4:
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
elif off == 2:
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
else:
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
return y.bitcast(fx.Float32) if is_f else y
def xred(v, off, op):
"""op(v, lane ^ off) for a symmetric op; 32 / 16 take both halves of one swap."""
if off < 16:
return op(v, xshfl(v, off))
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
a, b = permlane_swap(off, x, x)
if is_f:
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
return op(type(v)(a), type(v)(b))
def _add(a, b):
return a + b
def butterfly(v, offsets, op=_add):
"""Butterfly reduction over the given xor offsets, in the given order.
The order is the numerical contract: the aiter per-head norm reduces 32
lanes with offsets 16, 8, 4, 2, 1, and matching it keeps rstd bit-exact."""
for off in offsets:
v = xred(v, off, op)
return v
def traced(fn):
"""Apply flydsl's AST rewriting to a helper, so its ``while`` / ``if`` over
traced values lower to scf ops. Kernels rewrite only their own body."""
return ASTRewriter.transform(fn)
LOAD_BATCH = 4 # rounds of a strided loop whose loads go out together
@traced
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
"""A CTA's strided loop over elements v = tid + THREADS i < n (n a Python
int): ``load(v)`` of ``batch`` rounds issued before any ``use(v, loaded)``.
A round whose ``use`` stores to global memory keeps the next round's loads
behind it (they may alias), so the rounds' loads would go out in series. A
round past n loads element n - 1 (in range) and uses nothing."""
rounds = -(-n // THREADS)
for b0 in range_constexpr(0, rounds, batch):
idx = range(b0, min(b0 + batch, rounds))
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
for i, g in zip(idx, got):
v = tid + THREADS * i
if v < n:
use(v, g)
def mfma_bf16(a, b, c):
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
def row_sum(red, r, t, waves=None):
"""Row r, token t of the waves' 16x16 MFMA accumulators (wave w's at ``red``
+ (w * 64 + lane) * 4), summed in wave order from 0 (``waves``: which, in
order; all of them by default)."""
tot = fx.Float32(0.0)
for w in waves if waves is not None else range(WAVES):
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
return tot
def ballot(pred):
"""The wave's lanes where ``pred`` holds, a bit a lane."""
return fx.Int64(rocdl.ballot(T.i64, as_ir_value(pred)))
def popcount(x):
return fx.Int32(fx.ctpop(x))
def lanes_below(bal, lane):
"""How many of the lanes in ballot ``bal`` are below ``lane``."""
return popcount(bal & ((fx.Int64(1) << fx.Int64(lane)) - 1))
def load_ptr64(table, i):
"""Entry ``i`` of an i64 table in memory, wave-uniform (each half through
``uniform``): a peer's buffer base, a kernel argument block's pointer."""
pw = fx.Vector(bo.buffer_load(rsrc(table), i * 2, vec_width=2, dtype=T.i32))
return (fx.Int64(uniform(pw[1])) << 32) | fx.Int64(fx.Uint32(uniform(pw[0])))
def bf16_round(a):
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
def bf16_pair(a, b):
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
return (
fx.Vector.from_elements([a, b], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Float32)[0]
)
def bf_lo(w):
"""The f32 of a packed bf16 pair word's low half."""
return (w << 16).bitcast(fx.Float32)
def bf_hi(w):
"""The f32 of a packed bf16 pair word's high half."""
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
def bf2_f32(w):
"""Packed bf16 pair word -> (f32 low, f32 high)."""
return bf_lo(w), bf_hi(w)
def fp8_pack4(a, b, c, d):
"""Four f32 (already in range) -> one dword of four E4M3 bytes, RNE."""
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
# ---------------------------------------------------------------- TP ranks
def peer_bases(peers, tp):
"""Every rank's peer buffer base from the ``peers`` table (an i64 a rank),
loaded once: a put's address must not wait on a load of its own."""
return [load_ptr64(peers, p) for p in range(tp)]
def sum_partials(poll, own, pair_of, tp):
"""The ``tp`` ranks' bf16 pair partials at ``pair_of(src)`` of the mailbox
region ``own``, summed in fp32 in rank order 0 .. tp - 1: one poll batch ->
(lo sum, hi sum). Every rank sums in the same order, so they agree."""
ws = poll([(own, pair_of(src), 1) for src in range(tp)])
acc0, acc1 = bf2_f32(ws[0][0])
for src in range_constexpr(1, tp):
lo, hi = bf2_f32(ws[src][0])
acc0 = acc0 + lo
acc1 = acc1 + hi
return acc0, acc1
# ---------------------------------------------------------------- timeline stamps
def stamp(on, tls, tid, k):
# compile-time gate outside, traced condition inside: they cannot be one `and`
if const_expr(on): # noqa: SIM102
if tid == 0:
fx.ptr_store(memrealtime(), tls + k)
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/runtime/consensus.py
# atom/mono/runtime/peer_memory.py
"""Symmetric peer memory for the in-kernel TP all-reduces, and the agreement every
rank must reach before anything collective (a rank that turned the mono layer
off alone would leave the others spinning on the GPU)."""
import torch
import torch.distributed as dist
from aiter.ops.flydsl.quick_allreduce_int4_ipc import UncachedIpcHeap
# ---------------------------------------------------------------- TP agreement
class MonoUnsupported(Exception):
"""The loaded model or runtime configuration is outside what mono serves."""
def tp_agree(ok: bool, group) -> bool:
"""True when every rank of ``group`` passed ``ok`` true. Every rank must call
it at the same point; ``group`` is a CPU (gloo) process group, or None for a
single rank."""
if group is None:
return ok
flag = torch.tensor([int(ok)], dtype=torch.int32)
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group)
return bool(flag.item())
# ---------------------------------------------------------------- peer buffers
class _DeviceBytes:
"""``__cuda_array_interface__`` of ``nbytes`` bytes at a raw device address, for a
non-owning torch view."""
def __init__(self, ptr: int, nbytes: int):
self.__cuda_array_interface__ = {
"shape": (nbytes,),
"typestr": "|u1",
"data": (ptr, False),
"version": 3,
}
class PeerBuffer:
"""One uncached buffer per rank, every rank holding every peer's address
(``addresses``, int64, indexed by TP rank). What lives where, and why no rank
overwrites a peer's unread data, is the model's layout's to say.
The constructor is collective over ``group`` (the handle exchange, then an
agreement that every rank mapped every peer): every rank must reach it, so a
runner reaches it only after ``tp_agree``. A failed mapping on any rank raises
``MonoUnsupported`` on all of them.
"""
def __init__(
self,
nbytes: int,
group,
rank: int,
npes: int,
device: torch.device,
):
self.local = UncachedIpcHeap.alloc_uncached(nbytes)
# a view, not an owner: close() frees the memory under it
self.bytes = torch.as_tensor(_DeviceBytes(self.local, nbytes), device=device)
self.rank = rank
self._opened: list[int] = []
addresses = [self.local]
if npes > 1:
handles = UncachedIpcHeap.gather_object_list_via_broadcast(
group, UncachedIpcHeap.get_mem_handle_bytes(self.local)
)
addresses, failure = [], None
try:
for peer, handle in enumerate(handles):
if peer == rank:
addresses.append(self.local)
else:
base = UncachedIpcHeap.open_mem_handle(handle)
self._opened.append(base)
addresses.append(base)
except RuntimeError as err: # every rank must still reach the agreement
failure = err
if not tp_agree(failure is None, group):
self.close()
raise MonoUnsupported(
"peer memory handshake failed on "
f"{'this' if failure else 'another'}"
" TP rank"
) from failure
self.addresses = torch.tensor(addresses, dtype=torch.int64, device=device)
def close(self) -> None:
"""Close the peer mappings and free the local buffer; idempotent. Only once no
kernel of any rank can still touch it (the runner is being dropped)."""
opened, self._opened = self._opened, []
for base in opened:
UncachedIpcHeap.close_mem_handle(base)
if self.local:
self.bytes = None
UncachedIpcHeap.free_device_mem(self.local)
self.local = 0
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/plan/execution.py
# atom/mono/plan/layout.py
# atom/mono/plan/arith.py
# atom/mono/plan/shard.py
# atom/mono/plan/build_key.py
"""Plain-Python planning of the mono kernels: the execution model (one CTA a CU,
all resident), mailbox layouts, arithmetic that runs on ints and traced values
alike, TP shards, and build keys (every build parameter, and this package's
sources, in FlyDSL's JIT cache key)."""
import enum
import functools
import hashlib
from dataclasses import dataclass, fields
from pathlib import Path
# ---------------------------------------------------------------- execution model
BLOCKS = 256
THREADS = 512
WAVES = THREADS // 64
def first_task(bid, base):
"""CTA ``bid``'s first task of a stage placed from CTA ``base`` (any count of
CTAs past 0: placements that add task counts run past the grid), then every
BLOCKS-th."""
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
# ---------------------------------------------------------------- mailbox layouts
PAIR_BYTES = 8 # a (value, tag) pair
def pair_layout(items, align: int = PAIR_BYTES, start: int = 0) -> dict:
"""``items`` (name, pairs) laid out in order: name -> (byte offset, bytes),
each region from an ``align``-byte boundary at or past ``start``."""
out, off = {}, start
for name, pairs in items:
off = (off + align - 1) // align * align
out[name] = (off, pairs * PAIR_BYTES)
off += pairs * PAIR_BYTES
return out
# ---------------------------------------------------------------- arithmetic
def sel(pred, a, b):
"""A if pred else b: a conditional on Python values, ``select`` on traced
ones (rewrapped in the traced operand's type)."""
if isinstance(pred, bool):
return a if pred else b
out = pred.select(a, b)
traced = [x for x in (a, b) if not isinstance(x, int)]
return type(traced[0])(out) if traced else out
def imin(a, b):
return sel(a < b, a, b)
def i32(x):
"""``x``, asserted inside the device's Int32 range when it is a Python int."""
if isinstance(x, int):
assert -(2**31) <= x < 2**31, x
return x
def cdiv(a, b):
"""ceil(a / b) for a >= 0, b > 0."""
return i32((a + b - 1) // b)
# ---------------------------------------------------------------- TP shards
class ShardError(ValueError):
"""A width the TP size does not divide, or a TP size the model refuses."""
@dataclass(frozen=True)
class Shard:
tp: int
def __post_init__(self):
if self.tp < 1:
raise ShardError(f"TP {self.tp}")
def split(self, full: int, what: str) -> int:
"""``full`` / tp, refusing a remainder: a rank's share of ``what``."""
if full % self.tp:
raise ShardError(f"{what} {full} is not divisible by TP {self.tp}")
return full // self.tp
def padded(n: int, align: int) -> int:
return cdiv(n, align) * align
# ---------------------------------------------------------------- build keys
_SCALARS = (int, float, bool, str, enum.Enum)
_ROOT = Path(__file__).resolve().parents[1] # the mono package
@functools.cache
def source_digest(*paths: str) -> str:
"""sha256 of every ``.py`` file under ``paths`` (files or directories,
relative to the mono package), each file's path and contents: one model's
kernel sources, so an edit to another model's rebuilds none of these."""
h = hashlib.sha256()
for rel in paths:
root = _ROOT / rel
files = [root] if root.is_file() else sorted(root.rglob("*.py"))
if not files:
raise FileNotFoundError(f"no kernel sources at {root}")
for f in files:
h.update(str(f.relative_to(_ROOT)).encode())
h.update(f.read_bytes())
return h.hexdigest()[:16]
def key_tuple(key, sources: str) -> tuple:
"""``key``'s (field name, value) pairs, every value a scalar, then
("sources", ``sources``): the model's ``source_digest``."""
out = []
for f in fields(key):
value = getattr(key, f.name)
if not isinstance(value, _SCALARS):
raise TypeError(
f"{type(key).__name__}.{f.name} = {value!r}: a build parameter must be"
" a scalar to reach the JIT cache key"
)
out.append((f.name, value))
out.append(("sources", sources))
return tuple(out)
@@ -0,0 +1,177 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/sync.py
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
between GPUs (system scope), every address a named region (``Addr``)."""
from dataclasses import dataclass
from enum import Enum
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import (
CM_DEV,
CM_SYS,
bf16_pair,
rsrc,
traced,
)
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
@traced
def _spin(load_all, pending):
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
loop keeps the loads from being hoisted."""
v = load_all()
while pending(v):
rocdl.s_sleep(1)
v = load_all()
return v
class Space(Enum):
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
@dataclass(frozen=True)
class Addr:
"""A mailbox region's address: the region's name and space, and the traced
address. The space sets the hand-off's scope: device for SCRATCH, system for
PEER."""
region: str
space: Space
value: object
def sreg(base, offset, region):
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
def preg(base, offset, region):
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
``base``."""
return Addr(region, Space.PEER, base + fx.Int64(offset))
# the store / load cache policy of a hand-off in each space
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
class Mailbox:
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
Every 32-bit value is stored next to this launch's tag; a consumer
polls the payload until all tags match, so a hand-off costs one round trip
(no store drain, no separate flag). The mono layer's launches tag their
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
never carries a live tag. Every address is an ``Addr``: its space picks the
scope.
"""
def __init__(self, tag):
self.tag = tag
def put(self, a, i, v):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
bo.buffer_store(
fx.Vector.from_elements([bits, self.tag], fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def put_bf(self, a, i, vs):
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
store."""
words = []
for j in range_constexpr(len(vs) // 2):
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a.value),
i,
cache_modifier=_SCOPE_CM[a.space],
)
def put_words(self, a, i, words):
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
not kept."""
vals = []
for w in words:
vals += [fx.Int32(w), self.tag]
bo.buffer_store(
fx.Vector.from_elements(vals, fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def poll(self, specs, batch=POLL_MAX):
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
all of one space.
All pairs are re-loaded together while any tag is stale, so a batch costs
one round trip after its last producer lands. Returns the value words."""
if const_expr(len(specs) == 0):
return []
if const_expr(len(specs) > batch):
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
spaces = {a.space for a, _, _ in specs}
assert len(spaces) == 1, f"one poll batch spans {spaces}"
cm = _SCOPE_CM[spaces.pop()]
bases = [a.value for a, _, _ in specs]
tag = self.tag
def load_all():
words = []
for b, (_, i, n) in zip(bases, specs):
w = fx.Vector(
bo.buffer_load(
rsrc(b),
fx.Int32(i) * 2,
vec_width=2 * n,
dtype=T.i32,
cache_modifier=cm,
)
)
words += [w[e] for e in range(2 * n)]
return fx.Vector.from_elements(words, fx.Int32)
nw = sum(2 * n for _, _, n in specs)
def pending(v):
bad = v[1] != tag
for e in range_constexpr(3, nw, 2):
bad = bad | (v[e] != tag)
return bad
v = _spin(load_all, pending)
outs, e = [], 0
for _, _, n in specs:
outs.append([v[e + 2 * q] for q in range(n)])
e += 2 * n
return outs
@traced
def publish(put, a, i, v, who):
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
not overtake the data it announces, and a workgroup-scope release fence does
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
mode: the workgroup's waves share one CU, the readers here do not)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if who:
put(a, i, v)
File diff suppressed because it is too large Load Diff
+464
View File
@@ -0,0 +1,464 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1-Flash mono decode layer: host side.
One ``DSV41MonoLayer`` a TP rank runs every mono layer of a decode step (M <= 48
rows): two persistent launches a layer (``layer``) from the layer's
inputs at the attention seam to its outputs at the next one -- the MoE's
reduced output, the residual after the FFN seam and that seam's mixes -- with
both TP all-reduces inside the kernels (symmetric peer memory). A layer whose
attention vLLM runs (``ffn``) takes one launch from its unreduced attention
output on. Every argument is a device pointer: the launches can be captured in a
HIP graph. The kernels move their mailbox epoch on themselves; every rank runs
the same launches, so the ranks' epochs agree.
"""
from __future__ import annotations
import os
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
import torch
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
from .common.plan import BLOCKS
from .layer import (
EPOCH_WORDS,
MAX_TOKENS,
MonoBuild,
build_mono_ffn,
build_mono_k1,
build_mono_k2,
peer_half_bytes,
scratch_bytes,
)
from .stages.dims import Dims as MoeDims
from .stages.moe_shape import EXPERTS
__all__ = ["BLOCKS", "MAX_TOKENS", "AttnWeights", "DSV41MonoLayer", "MonoLayerWeights"]
_LOG2E = 1.4426950408889634
HC = 4
def _ensure_writable_flydsl_cache() -> None:
"""Aiter points FlyDSL at its bundled, read-only cache; new kernels need a
writable one."""
cur = os.environ.get("FLYDSL_RUNTIME_CACHE_DIR")
if cur and os.access(cur, os.W_OK):
return
path = Path.home() / ".flydsl" / "cache"
path.mkdir(parents=True, exist_ok=True)
os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = str(path)
def _check_tensors(owner, tag: str, want: dict) -> None:
"""Each named tensor of ``owner`` has the (shape, dtype) the kernels read,
contiguous: their buffer loads are unbounded, so a mismatch would read
garbage rather than fault."""
for name, (shape, dtype) in want.items():
t = getattr(owner, name)
assert tuple(t.shape) == shape and t.dtype == dtype and t.is_contiguous(), (
f"{tag} {name}: {tuple(t.shape)} {t.dtype}, want {shape} {dtype}"
)
@dataclass
class AttnWeights:
"""One layer's attention tensors on this rank, in vLLM's loaded layout."""
layer_id: int
wqkv: torch.Tensor # [1792, 5120] e4m3: [wq_a; wkv]
wqkv_scale: torch.Tensor # [56, 160] uint8 E8M0 (32 x 32 blocks)
q_norm: torch.Tensor # [1280] bf16
kv_norm: torch.Tensor # [512] bf16
wq_b: torch.Tensor # [H * 512, 1280] e4m3
wq_b_scale: torch.Tensor # [H * 16, 40] uint8
wo_a: torch.Tensor # [G * 1024, 4096] e4m3
wo_a_scale: torch.Tensor # [G * 32, 128] uint8
wo_b: torch.Tensor # [5120, G * 1024] e4m3
wo_b_scale: torch.Tensor # [160, G * 32] uint8
attn_sink: torch.Tensor # [>= H] f32
cos_sin: torch.Tensor # [positions, 64] f32 (the layer's rope table)
ratio: int # 0 / 1 / 2: the layer's compress ratio
def check(self, d: Dims) -> None:
h, g = d.heads, d.groups
want = {
"wqkv": ((1792, HIDDEN), torch.float8_e4m3fn),
"wqkv_scale": ((56, HIDDEN // 32), torch.uint8),
"wq_b": ((h * HEAD_DIM, 1280), torch.float8_e4m3fn),
"wq_b_scale": ((h * HEAD_DIM // 32, 40), torch.uint8),
"wo_a": ((g * 1024, d.group_k), torch.float8_e4m3fn),
"wo_a_scale": ((g * 32, d.group_k // 32), torch.uint8),
"wo_b": ((HIDDEN, g * 1024), torch.float8_e4m3fn),
"wo_b_scale": ((HIDDEN // 32, g * 32), torch.uint8),
}
_check_tensors(self, f"layer {self.layer_id}", want)
assert self.cos_sin.dtype == torch.float32 and self.cos_sin.shape[-1] == 64
@dataclass
class MonoLayerWeights:
"""One layer's tensors on this rank, in vLLM's loaded layout (``attn``:
None for a layer whose attention vLLM runs)."""
attn: AttnWeights | None
hc_attn_fn: torch.Tensor # [24, 4 * 5120] f32
hc_attn_scale: torch.Tensor # [3] f32
hc_attn_base: torch.Tensor # [24] f32
attn_norm: torch.Tensor # [5120] bf16
hc_ffn_fn: torch.Tensor
hc_ffn_scale: torch.Tensor
hc_ffn_base: torch.Tensor
ffn_norm: torch.Tensor
gate_w: torch.Tensor # [384, 5120] bf16
bias: torch.Tensor # [384] f32 (e_score_correction_bias)
w13: torch.Tensor # [384, 2 inter, 2560] fp4x2, aiter (16, 16) shuffle
w13_s: torch.Tensor # its scales, aiter shuffle_scale order
w2: torch.Tensor # [384, 5120, inter / 2] fp4x2
w2_s: torch.Tensor
sgu: torch.Tensor # shared gate_up [2 inter, 5120] e4m3, row-major
sgu_s: torch.Tensor # [2 inter / 32, 160] E8M0
sw2: torch.Tensor # shared down [5120, inter] e4m3
sw2_s: torch.Tensor
def check(self, tp: int) -> None:
"""The MoE's tensors: 384 experts, TP-sharded intermediates (the routed
one padded as the loader pads it), AITER's A8W4 layout, 32 x 32 E8M0
blocks for the shared expert."""
d, fp4, e4m3, u8 = (
MoeDims(tp),
torch.float4_e2m1fn_x2,
torch.float8_e4m3fn,
torch.uint8,
)
want = {
"gate_w": ((EXPERTS, HIDDEN), torch.bfloat16),
"bias": ((EXPERTS,), torch.float32),
"w13": ((EXPERTS, 2 * d.inter, HIDDEN // 2), fp4),
"w13_s": ((EXPERTS * 2 * d.inter, HIDDEN // 32), u8),
"w2": ((EXPERTS, HIDDEN, d.inter // 2), fp4),
"w2_s": ((EXPERTS * HIDDEN, d.down_scale_cols), u8),
"sgu": ((2 * d.sh_inter, HIDDEN), e4m3),
"sgu_s": ((2 * d.sh_inter // 32, HIDDEN // 32), u8),
"sw2": ((HIDDEN, d.sh_inter), e4m3),
"sw2_s": ((HIDDEN // 32, d.sh_inter // 32), u8),
}
_check_tensors(self, "MoE", want)
class DSV41MonoLayer:
"""The mono decode runner of one TP rank (``group``: its TP group, for the
peer-memory handle exchange; a gloo / CPU group)."""
def __init__(self, tp: int, rank: int, group, device: torch.device | str = "cuda"):
_ensure_writable_flydsl_cache()
from .common.peer_memory import PeerBuffer
self.tp, self.rank = tp, rank
self.d = Dims(tp)
dev = self.device = torch.device(device)
self._scratch: dict[int, torch.Tensor] = {}
# [epoch, -, -, -, a mark per CTA, the MoE's counters]
self.epoch = torch.zeros(EPOCH_WORDS, dtype=torch.int32, device=dev)
self.q = torch.zeros(
MAX_TOKENS, self.d.heads, HEAD_DIM, dtype=torch.bfloat16, device=dev
)
self.kt = torch.zeros(MAX_TOKENS * KEYS, dtype=torch.int32, device=dev)
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self._first_pre[:, 0] = 1.0
# an index source's K1 outputs, vLLM's indexer's inputs
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
# the attention seam's outputs, K1 -> K2
self.res_mid = torch.zeros(
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
)
self.post_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self.comb_a = torch.zeros(MAX_TOKENS, HC, HC, dtype=torch.float32, device=dev)
self.pre_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self.peer = PeerBuffer(2 * peer_half_bytes(tp), group, rank, tp, dev)
self.peer.bytes.zero_()
self._qk_scale = (
torch.tensor([HEAD_DIM**-0.5 * _LOG2E], dtype=torch.float32)
.view(torch.int32)
.item()
)
self._kernels: dict = {}
def scratch(self, tokens: int) -> torch.Tensor:
"""Step width ``tokens``'s scratch: a width's layout has memory of its own
(``layer.scratch_layout``). Allocated at the width's first step, which is
eager: vLLM runs every graph's batch eagerly before capturing it."""
buf = self._scratch.get(tokens)
if buf is None:
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
f"DSv4.1 mono decode: step width {tokens} first reached inside "
"a CUDA graph capture"
)
buf = torch.zeros(
scratch_bytes(tokens, self.tp), dtype=torch.uint8, device=self.device
)
self._scratch[tokens] = buf
return buf
def first_pre_mix(self, tokens: int) -> torch.Tensor:
return self._first_pre[:tokens]
@staticmethod
def supports(tokens: int) -> bool:
return 1 <= tokens <= MAX_TOKENS
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
key = (tokens, ratio, fold, index)
if key not in self._kernels:
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
return self._kernels[key]
def ffn_kernel(self, tokens: int):
key = (tokens, "ffn")
if key not in self._kernels:
self._kernels[key] = build_mono_ffn(MonoBuild(tokens, self.tp, 0))
return self._kernels[key]
@staticmethod
def _outs(M: int, residual: torch.Tensor) -> tuple:
dev = residual.device
return (
torch.empty(M, HIDDEN, dtype=torch.bfloat16, device=dev),
torch.empty_like(residual),
torch.empty(M, HC, 1, dtype=torch.float32, device=dev),
torch.empty(M, HC, HC, dtype=torch.float32, device=dev),
torch.empty(M, HC, dtype=torch.float32, device=dev),
)
def forward(
self,
w: MonoLayerWeights,
x: torch.Tensor, # [M, 5120] bf16: the previous FFN output (reduced)
residual: torch.Tensor, # [M, 4, 5120] bf16
post_mix: torch.Tensor, # [M, 4, 1] f32
res_mix: torch.Tensor, # [M, 4, 4] f32
pre_mix: torch.Tensor, # [M, 4] f32
positions: torch.Tensor, # [M] int64
slot_mapping: torch.Tensor, # [M] int64
swa_cache: torch.Tensor, # [blocks, block, 584] uint8
swa_indices: torch.Tensor, # [>= M, 1, 128] int32
swa_lens: torch.Tensor, # [>= M] int32
token_to_req: torch.Tensor, # [>= M] int32
topk_indices: torch.Tensor | None = None,
comp_cache: torch.Tensor | None = None,
comp_block_table: torch.Tensor | None = None,
outs: tuple | None = None,
fold: bool = True,
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
):
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
source's top-k selection into ``topk_indices``, run between the
kernels on K1's normed input and qr."""
M = x.shape[0]
assert self.supports(M), M
a = w.attn
assert a is not None
ratio = a.ratio
index = select is not None
assert not index or ratio
k1, k2 = self.kernels(M, ratio, fold, index)
st = torch.cuda.current_stream()
if outs is None:
outs = self._outs(M, residual)
out, res_out, post_out, comb_out, pre_out = outs
for t_ in (x, residual, post_mix, res_mix, pre_mix, positions, slot_mapping):
assert t_.is_contiguous()
assert positions.dtype == slot_mapping.dtype == torch.int64
assert swa_indices.dtype == swa_lens.dtype == token_to_req.dtype == torch.int32
assert swa_indices.stride(0) == 128 and swa_cache.dtype == torch.uint8
assert swa_cache.shape[-1] == 584 and swa_cache.stride(1) == 584
swa_block = swa_cache.shape[1]
if ratio:
assert topk_indices is not None and comp_cache is not None
assert comp_block_table is not None
assert topk_indices.dtype == torch.int32 and topk_indices.stride(0) == 512
assert comp_cache.dtype == torch.uint8 and comp_cache.stride(1) == 584
assert comp_block_table.dtype == torch.int32
comp, comp_stride, comp_block = (
comp_cache,
comp_cache.stride(0),
comp_cache.shape[1],
)
bt, bt_stride, topk = (
comp_block_table,
comp_block_table.stride(0),
topk_indices,
)
else:
comp, comp_stride, comp_block = self._dummy, 0, 1
bt, bt_stride, topk = self._dummy, 0, self._dummy
k1(
residual.data_ptr(),
x.data_ptr(),
post_mix.data_ptr(),
res_mix.data_ptr(),
pre_mix.data_ptr(),
w.hc_attn_fn.data_ptr(),
w.hc_attn_scale.data_ptr(),
w.hc_attn_base.data_ptr(),
w.attn_norm.data_ptr(),
self.res_mid.data_ptr(),
self.post_a.data_ptr(),
self.comb_a.data_ptr(),
self.pre_a.data_ptr(),
a.wqkv.data_ptr(),
a.wqkv_scale.data_ptr(),
a.q_norm.data_ptr(),
a.kv_norm.data_ptr(),
a.wq_b.data_ptr(),
a.wq_b_scale.data_ptr(),
a.cos_sin.data_ptr(),
positions.data_ptr(),
slot_mapping.data_ptr(),
swa_cache.data_ptr(),
swa_cache.stride(0),
swa_block,
self.q.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
comp_block,
bt.data_ptr(),
bt_stride,
topk.data_ptr(),
token_to_req.data_ptr(),
self.kt.data_ptr(),
self.klen.data_ptr(),
self.normed.data_ptr(),
self.qr.data_ptr(),
self.scratch(M).data_ptr(),
self.epoch.data_ptr(),
0,
stream=st,
)
if select is not None:
select(self.normed[:M], self.qr[:M])
k2(
self.q.data_ptr(),
swa_cache.data_ptr(),
swa_cache.stride(0),
swa_block,
comp.data_ptr(),
comp_stride,
comp_block,
self.kt.data_ptr(),
self.klen.data_ptr(),
slot_mapping.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
token_to_req.data_ptr(),
topk.data_ptr(),
bt.data_ptr(),
bt_stride,
positions.data_ptr(),
a.attn_sink.data_ptr(),
self._qk_scale,
a.cos_sin.data_ptr(),
a.wo_a.data_ptr(),
a.wo_a_scale.data_ptr(),
a.wo_b.data_ptr(),
a.wo_b_scale.data_ptr(),
self._zero_rec.data_ptr(),
self.res_mid.data_ptr(),
self.post_a.data_ptr(),
self.comb_a.data_ptr(),
self.pre_a.data_ptr(),
w.hc_ffn_fn.data_ptr(),
w.hc_ffn_scale.data_ptr(),
w.hc_ffn_base.data_ptr(),
w.ffn_norm.data_ptr(),
res_out.data_ptr(),
post_out.data_ptr(),
comb_out.data_ptr(),
pre_out.data_ptr(),
w.gate_w.data_ptr(),
w.bias.data_ptr(),
w.w13.data_ptr(),
w.w13_s.data_ptr(),
w.w2.data_ptr(),
w.w2_s.data_ptr(),
w.sgu.data_ptr(),
w.sgu_s.data_ptr(),
w.sw2.data_ptr(),
w.sw2_s.data_ptr(),
out.data_ptr(),
self.scratch(M).data_ptr(),
self.peer.local,
self.peer.addresses.data_ptr(),
self.rank,
self.epoch.data_ptr(),
0,
stream=st,
)
return outs
def ffn(
self,
w: MonoLayerWeights,
part: torch.Tensor, # [M, 5120] bf16: this rank's unreduced wo_b output
residual: torch.Tensor, # [M, 4, 5120] bf16, after the attention seam
post_mix: torch.Tensor, # [M, 4, 1] f32, the attention seam's
res_mix: torch.Tensor, # [M, 4, 4] f32
pre_mix: torch.Tensor, # [M, 4] f32
outs: tuple | None = None,
):
"""-> (out, residual, post_mix, res_mix, pre_mix) of a layer whose
attention vLLM ran: the attention's TP reduce, the FFN seam, the MoE and
its all-reduce in one launch."""
M = part.shape[0]
assert self.supports(M), M
for t_ in (part, residual, post_mix, res_mix, pre_mix):
assert t_.is_contiguous()
assert part.dtype == residual.dtype == torch.bfloat16
if outs is None:
outs = self._outs(M, residual)
out, res_out, post_out, comb_out, pre_out = outs
self.ffn_kernel(M)(
part.data_ptr(),
residual.data_ptr(),
post_mix.data_ptr(),
res_mix.data_ptr(),
pre_mix.data_ptr(),
w.hc_ffn_fn.data_ptr(),
w.hc_ffn_scale.data_ptr(),
w.hc_ffn_base.data_ptr(),
w.ffn_norm.data_ptr(),
res_out.data_ptr(),
post_out.data_ptr(),
comb_out.data_ptr(),
pre_out.data_ptr(),
w.gate_w.data_ptr(),
w.bias.data_ptr(),
w.w13.data_ptr(),
w.w13_s.data_ptr(),
w.w2.data_ptr(),
w.w2_s.data_ptr(),
w.sgu.data_ptr(),
w.sgu_s.data_ptr(),
w.sw2.data_ptr(),
w.sw2_s.data_ptr(),
out.data_ptr(),
self.scratch(M).data_ptr(),
self.peer.local,
self.peer.addresses.data_ptr(),
self.rank,
self.epoch.data_ptr(),
stream=torch.cuda.current_stream(),
)
return outs
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The mono layer's kernel sources, in every build's JIT cache key
(``common.plan.key_tuple``): an edit anywhere in this package rebuilds."""
from vllm.models.deepseek_v41.amd.mono.common.plan import source_digest
SOURCES = source_digest(".")
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The V4.1 stages the mono layer adapts from ROCm/ATOM: the mHC seams
(``seam``), the MoE (``moe``) and the GEMV / LDS helpers they share (``gemv``)."""
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/dims.py
"""A TP rank's share of V4.1's MoE widths: every constant the MoE stages derive
from the TP size, in one place (a build key's ``tp`` picks it)."""
from dataclasses import dataclass
from vllm.models.deepseek_v41.amd.mono.common.plan import Shard, padded
# the model's full widths
MOE_INTER = 2304 # a routed expert's intermediate width
SHARED_INTER = 2304 # the shared expert's
# the loader pads a routed expert's rank share to a multiple of this (``FusedMoE``
# ``pad_align``) with zero weights
MOE_PAD = 128
ROWS = 16 # GEMV rows a task
UG_PART = 32 # intermediate columns an ug task part: one FP4 group
@dataclass(frozen=True)
class Dims:
tp: int
def __post_init__(self):
shard = Shard(self.tp)
shard.split(MOE_INTER, "routed intermediate")
shard.split(SHARED_INTER, "shared intermediate")
@property
def inter_real(self) -> int:
"""A routed expert's real intermediate width on this rank."""
return MOE_INTER // self.tp
@property
def inter(self) -> int:
"""... padded as the loader pads it (the rest zero weights)."""
return padded(self.inter_real, MOE_PAD)
@property
def sh_inter(self) -> int:
"""The shared expert's intermediate width on this rank."""
return SHARED_INTER // self.tp
@property
def shared_tasks(self) -> int:
return self.sh_inter // ROWS
@property
def mid_words(self) -> int:
"""MXFP8 words of a pick's routed intermediate."""
return self.inter // 4
@property
def down_scale_cols(self) -> int:
"""w2_s's e8m0 columns: inter / 32, padded to a multiple of 8
(aiter ``shuffle_scale``)."""
return padded(self.inter // 32, 8)
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (its GEMV and LDS helpers).
"""The FP8 GEMV tiles of the seam and MoE stages and the helpers they share:
token tiles, LDS rows of MXFP8 words, scalar loads and the mailbox-to-LDS
copy."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, range_constexpr
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.mx import (
UNIT_SCALE,
mfma_scaled,
)
from vllm.models.deepseek_v41.amd.mono.common.ops import (
CM_NT,
fresh,
rsrc,
traced,
)
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
from vllm.models.deepseek_v41.amd.mono.common.sync import POLL_MAX
ROWS = 16 # GEMV rows a task
TILE = 16 # an MFMA's N: the tokens a GEMV pass takes (``token_tiles``)
def ld_f32(ptr, i):
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
def ld_bf(ptr, i):
return fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
)
def token_tiles(s):
"""A step's tokens in GEMV passes: [(first token, tokens)]. A GEMV loads its
weights once and runs every pass on them, an x tile in LDS at a time."""
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
def tile_rows(s):
"""The LDS rows an x tile holds."""
return min(s, TILE)
# a token's LDS row of MXFP8 words / E8M0 codes, 16 of which a MFMA B operand
# gathers, is padded by 16 B: at a multiple of 64 words they share a bank group
LDS_PAD = 4
def lds_row(words):
return words + LDS_PAD
def lds_at(u, row):
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
return u + u // row * LDS_PAD
def lds_tail(k):
"""Words a K = ``k`` row's last K step reads past its ``lds_row`` (selected
out): the last row's buffer must hold them."""
return max(0, 32 * -(-k // 128) - lds_row(k // 4))
def plus(t0, v):
"""Token ``v`` of the tile from ``t0`` (a first tile adds nothing; ``t0``
traced: a tile a CTA picks at run time)."""
return v if isinstance(t0, int) and t0 == 0 else v + t0
# ---------------------------------------------------------------- GEMV
def _fp8_step_load(c, r_w, r_ws, k, rg, st, row_major=False):
"""K step st's global operands of rows 16 rg ..: the two 64-column weight
chunks (the upper clamped into the row) and the raw scale word. The (16, 16)
shuffle lays a row group out in 32-column blocks of 16 rows, a chunk two of
them (lanes 0-31, 32-63); ``k`` % 64 = 32 (a last quarter step): the lanes
of a block past the row read its last block. ``row_major`` (vLLM's dense
layout, k % 64 == 0): the same registers from a row-major [N, K] weight --
lane l row 16 rg + l % 16, K bytes 128 st + 16 (l // 16) .. and + 64."""
lane = c["lane"]
j = lane // 16
kg = k // 32
if const_expr(row_major):
assert k % 64 == 0
row = rg * 16 + lane % 16
lo_at = row * (k // 4) + st * 32 + j * 4
hi_at = row * (k // 4) + fx.min(2 * st + 1, k // 64 - 1) * 16 + j * 4
lo = fx.Vector(
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
hi = fx.Vector(
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
if const_expr(k % 64 == 0):
lo_at = (rg * (k // 64) + 2 * st) * 256 + lane * 4
else:
lo_at = rg * (k * 4) + fx.min(4 * st + lane // 32, kg - 1) * 128 + lane % 32 * 4
lo = fx.Vector(
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
if const_expr(k % 64 == 0):
hi_at = (rg * (k // 64) + fx.min(2 * st + 1, k // 64 - 1)) * 256 + lane * 4
else:
hi_at = rg * (k * 4) + fx.min(2 * st + 1, k // 64 - 1) * 256 + lane * 4
hi = fx.Vector(
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
def _fp8_step_mfma(c, k, xl, xsl, st, ops, acc, live=None, rows=None):
"""``acc`` through K step st's scaled MFMA: ``_fp8_step_load``'s operands
against every token's MXFP8 row in LDS. ``live`` false: a step past the end
(its operands those of the last step), zero at unit scale. ``k`` % 64 = 32:
a lane's lower block past the row is zero on both sides."""
s, lane = c["S"] if rows is None else rows, c["lane"]
lo, hi, sa = ops
kg = k // 32
x_row, s_row = lds_row(k // 4), lds_row(kg)
j = lane // 16
col = fx.min(lane % 16, s - 1)
if const_expr(live is not None):
st = fx.min(st, (k + 127) // 128 - 1)
lo = [live.select(lo[d], fx.Int32(0)) for d in range(4)]
past = (2 * st + 1) * 64 >= k
if const_expr(live is not None):
past = past | ~live
av = fx.Vector.from_elements(
[lo[d] for d in range(4)] + [past.select(fx.Int32(0), hi[d]) for d in range(4)],
fx.Int32,
)
grp = st * 4 + j
xlo = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xhi = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + 16 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
if const_expr(k % 64 != 0):
over = 4 * st + j // 2 >= kg
av = fx.Vector.from_elements(
[over.select(fx.Int32(0), av[d]) for d in range(4)]
+ [av[4 + d] for d in range(4)],
fx.Int32,
)
xlo = [over.select(fx.Int32(0), xlo[d]) for d in range(4)]
xv = fx.Vector.from_elements(
[xlo[d] for d in range(4)]
+ [past.select(fx.Int32(0), xhi[d]) for d in range(4)],
fx.Int32,
)
sb = fx.ptr_load(xsl + (col * s_row + fx.min(grp, kg - 1)))
unit = grp >= kg
if const_expr(live is not None):
unit = unit | ~live
sa = unit.select(fx.Int32(UNIT_SCALE), sa)
sb = unit.select(fx.Int32(UNIT_SCALE), sb)
return mfma_scaled(av, xv, acc, sa, sb)
def _gemv_steps(c, k, split, tiled=False):
"""This wave's K steps (a step past the end marked, ``split`` waves past
split repeat the last one's; ``tiled``: wave w takes part w % split of tile
w / split): [(step, live)]."""
wave = c["wave"]
steps = (k + 127) // 128
if const_expr(split is None):
per = (steps + WAVES - 1) // WAVES
ragged = steps % WAVES != 0
return [
(wave + WAVES * i, (wave + WAVES * i < steps) if ragged else None)
for i in range(per)
]
assert split <= WAVES and steps % split == 0
per = steps // split
part = wave % split if tiled else fx.min(wave, split - 1)
return [(part * per + i, None) for i in range(per)]
def gemv_fp8_loads(c, w, ws, k, rg, split=None, tiled=False, row_major=False):
"""The weight operands of ``gemv_fp8``'s rows for this wave, every one in
flight at once (a step's latency each would put them in series): loads
only. A step past the end loads the last step. ``tiled`` (split x tiles =
WAVES): row groups rg .., one a group of ``split`` waves."""
steps = (k + 127) // 128
r_w, r_ws = rsrc(w), rsrc(ws)
if const_expr(tiled):
assert WAVES % split == 0
rg = rg + c["wave"] // split
return [
_fp8_step_load(
c,
r_w,
r_ws,
k,
rg,
st if live is None else fx.min(st, steps - 1),
row_major,
)
for st, live in _gemv_steps(c, k, split, tiled)
]
def gemv_fp8_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=None):
"""``gemv_fp8_loads``' operands against the rows in LDS (``rows``: a token
tile's, every token's by default) ->
``red[wave][lane]``; the scaled 16x16x128 MFMA, chunk kc in dwords 0-3 and kc +
1 in 4-7. ``k`` a multiple of 32: a last half step has its upper chunk zeroed
on both sides at unit scale (a NaN byte past the row must not reach the
MFMA), its loads clamped into the row: the last row group's would read past
the tensor; a last quarter step its lower chunk's upper block too. A step
past the end adds zero at unit scale.
The K order is the original kernel's, which ``row_sum`` completes:
wave w takes steps w, w + 8, ... (``split`` None: aiter's preshuffled group32
GEMM, P0.5 microbenchmark 1), or waves 0 .. split - 1 one contiguous K range
each, the other waves zero (aiter's split-K bmm, whose last split sums the
partials in split order)."""
lane, wave = c["lane"], c["wave"]
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i, (st, live) in enumerate(_gemv_steps(c, k, split, tiled)):
acc = _fp8_step_mfma(c, k, xl, xsl, st, ops[i], acc, live, rows)
if const_expr(split is not None and not tiled):
acc = fx.Vector.from_elements(
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
fx.Float32,
)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
# ---------------------------------------------------------------- mailbox -> LDS
@traced
def poll_copy(c, idx, width, spans, live=None):
"""Mailbox spans [(region, pair_of, n, dst, row)] into LDS: element u (pair
``pair_of(u)``) to dst + u, or ``lds_at(u, row)`` for a span of rows
``row`` words wide. Thread ``idx`` of ``width`` takes u = idx, idx +
width, ..., POLL_MAX a round trip, each batch stored before the next is
polled (holding them all spills past S = 16); a thread past a span's end
re-reads its last element and stores nothing.
``live``: a span's elements the reader needs (traced, <= n), a span each;
a batch past them is not polled and an element past them neither polled
nor written (one past a step's rows is never written)."""
idx = fresh(idx) # its elements' indices not hoisted out of a task loop
if const_expr(live is None):
_poll_store_batches(
c,
[
(region, pair_of, n, (dst, row), idx + width * i)
for region, pair_of, n, dst, row in spans
for i in range((n + width - 1) // width)
],
)
return
for (region, pair_of, n, dst, row), need in zip(spans, live):
chunks = [
(region, pair_of, need, (dst, row), idx + width * i)
for i in range((n + width - 1) // width)
]
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
if chunks[b0][4] < need:
_poll_store_batches(c, chunks[b0 : b0 + POLL_MAX])
@traced
def _poll_store_batches(c, chunks):
"""``poll_copy``'s chunks [(region, pair_of, n, (dst, row), u)], POLL_MAX a
round trip, each batch stored before the next is polled."""
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
batch = chunks[b0 : b0 + POLL_MAX]
got = c["poll"]([(r, p(fx.min(u, n - 1)), 1) for r, p, n, _, u in batch])
for (_, _, n, (dst, row), u), v in zip(batch, got):
at = u if row is None else lds_at(u, row)
if u < n:
fx.ptr_store(v[0], dst + at)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/moe_plan.py
"""Every per-step decision of which MoE ug flags (UGF) and intermediate rows
(MID) the ug stage writes and the down stage reads, defined once.
The route table (``stage_route``) gives each expert with picks a slot of U, its
picks a run from ``slot_first``, and, past one token tile, its picks tiles of
``TILE`` from ``tile_first`` (each tile's slot ``tile_slot``). An ug task covers
a unit -- a slot, or past one token tile a pick tile -- and one of ``ug_parts`` parts
of the intermediate; its flag is ``ug_flag``. A down round polls each of its
slots' flags (``polled_tile`` its tiles) before it reads their MID rows. Called
by the kernel on traced Int32 and by the CPU property test on Python ints.
"""
from vllm.models.deepseek_v41.amd.mono.common.plan import cdiv, imin
TILE = 16 # picks a pick tile: an MFMA's N
def pick_tiles(count):
"""A slot's pick tiles: its ``count`` picks, ``TILE`` a tile."""
return cdiv(count, TILE)
def ug_parts(groups, ug_groups):
"""A unit's ug parts: the expert's ``groups`` FP4 groups of its real width,
``ug_groups`` a part. A part wholly in the padding is not run: MID's padded
columns stay the step's zeros, against zero weights in down."""
return cdiv(groups, ug_groups)
def ug_flag(unit, part, ug_parts):
"""The UGF pair of ug task (``unit``, ``part``), ``ug_parts`` parts a unit."""
return unit * ug_parts + part
def ug_tasks(units, ug_parts):
"""A step's ug tasks: every part of each of its ``units``."""
return units * ug_parts
def tile_pick0(slot_first, tile, tile_first):
"""Pick tile ``tile``'s first pick, its slot's picks from ``slot_first`` and
tiles from ``tile_first``."""
return slot_first + (tile - tile_first) * TILE
def polled_tile(tile_first, tiles, j):
"""The pick tile a down round polls as a slot's ``j``-th, the slot's
``tiles`` tiles from ``tile_first``: the last past them."""
return tile_first + imin(j, tiles - 1)
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/moe_shape.py
"""The MoE stages' build key and route shape: what they are built for (``MoeBuild``),
and the task counts, placements and LDS route table that follow from its
routing (``RouteShape``), with the table's readers. Plain Python but for the
readers; the kernel's stages are in ``moe``."""
from dataclasses import dataclass, field
import flydsl.expr as fx
from vllm.models.deepseek_v41.amd.mono.common.plan import BLOCKS, WAVES
from vllm.models.deepseek_v41.amd.mono.stages import moe_plan as mplan
from vllm.models.deepseek_v41.amd.mono.stages.dims import UG_PART, Dims
from vllm.models.deepseek_v41.amd.mono.stages.gemv import ROWS, TILE, token_tiles
from vllm.models.deepseek_v41.amd.mono.stages.router import PARTS as ROUTER_PARTS
EXPERTS, TOPK = 384, 6 # the target's routing (a build's: ``MoeBuild``)
ROUTER0 = 8 # the router's first task: past the xq tasks (a token each)
MASK_WORDS = 2 # an expert's token bitmap in the route table: tokens <= 64
# a route lane's experts (EXPERTS / 64) sorted descending, ties kept in index
# order: the network by their count
SORT_NETS = {
6: [
(0, 1),
(2, 3),
(4, 5),
(0, 2),
(1, 4),
(3, 5),
(0, 1),
(2, 3),
(4, 5),
(1, 2),
(3, 4),
(2, 3),
],
2: [(0, 1)],
}
@dataclass(frozen=True)
class MoeBuild:
tokens: int
# routed experts and a token's top-k: the target's, or the DSpark draft's
experts: int = EXPERTS
topk: int = TOPK
# the TP size: every width a rank holds (``Dims``)
tp: int = 4
timeline: bool = field(default=False, metadata={"sym": "tl"})
# FP4 groups an ug task (``RouteShape.ug_groups``)
ug_groups: int = 2
@dataclass(frozen=True)
class RouteShape:
"""A build's routing: ``experts`` routed experts, ``topk`` a token of
``tokens``, the task counts and placements that follow, and the route table
``load_route`` lays out in LDS: every token's (ids, weights), |U|, the
experts of U, each (t, k)'s slot in U, each expert's slot, the picks in slot
order and each slot's first position in it (past one token tile also each
slot's first pick tile and each pick tile's slot)."""
experts: int
topk: int
tokens: int
tp: int
# FP4 groups (32 intermediate columns) an ug task, its waves splitting K in
# halves (2) or quarters (1). 1 is twice the tasks, so the CTAs' ug ends
# closer together: faster at the draft's widths (40 rows -4 us), slower at
# the target's (48 rows +3 us; the runners pick)
ug_groups: int = 2
@property
def dims(self) -> Dims:
return Dims(self.tp)
@property
def ug_parts(self) -> int:
"""An expert's ug tasks (``moe_plan.ug_parts``)."""
return mplan.ug_parts(self.dims.inter_real // UG_PART, self.ug_groups)
@property
def ug_k_slices(self) -> int:
"""The K slices of an ug task's waves: a wave a (group, 16-column half,
slice)."""
return WAVES // (2 * self.ug_groups)
@property
def picks(self) -> int:
return self.tokens * self.topk
@property
def max_u(self) -> int:
return min(self.picks, self.experts)
@property
def router_groups(self) -> tuple[int, int]:
"""``stage_router``'s token groups and their tokens: a group a token
tile, as many as one round of CTAs holds (a CTA running two serialized
them: the route waited on the 32 that did at 48 rows)."""
per_group = self.experts // ROWS * ROUTER_PARTS
groups = max(1, min(len(token_tiles(self.tokens)), BLOCKS // per_group))
tokens = -(-self.tokens // groups)
assert (groups - 1) * tokens < self.tokens
return groups, tokens
@property
def router_tasks(self) -> int:
# (token group, 16-row tile, K part): ``stage_router``
return self.router_groups[0] * self.experts // ROWS * ROUTER_PARTS
@property
def router0(self) -> int:
"""Past the xq tasks (a token each from CTA 0)."""
return max(ROUTER0, self.tokens)
@property
def route0(self) -> int:
return self.router0 + self.router_tasks
@property
def shared0(self) -> int:
return self.route0 + max(8, self.tokens)
@property
def shared_units(self) -> int:
"""The shared expert's (task, token tile) units (``stage_shared``)."""
return self.dims.shared_tasks * len(token_tiles(self.tokens))
@property
def ug0(self) -> int:
"""The first ug round's placement: the shared units' CTAs last."""
return self.shared0 + self.shared_units
@property
def sort_net(self):
return SORT_NETS[self.experts // 64]
@property
def tab_words(self) -> int:
return self.tile_slot0 + self.max_tiles if self.multi else self.queue0 + 1
@property
def multi(self) -> bool:
"""More than one token tile: an ug task is a (tile of a slot's picks,
part) (``stage_ug``)."""
return self.tokens > TILE
@property
def slot_tiles(self) -> int:
"""A slot's pick tiles at most (a token picks an expert once)."""
return len(token_tiles(self.tokens))
@property
def max_tiles(self) -> int:
"""The pick tiles of U at most: a slot's partial tile each, then the
whole tiles, and never more than a tile a pick."""
return min(self.picks, self.max_u + self.picks // TILE)
@property
def tile_first0(self) -> int:
"""Past one token tile: each slot's first pick tile, then their count."""
return self.queue0 + 1
@property
def tile_slot0(self) -> int:
"""Past one token tile: each pick tile's slot of U."""
return self.tile_first0 + self.max_u + 1
@property
def queue0(self) -> int:
"""The ug task this CTA took from the queue (``take_ug_task``)."""
return self.mask0 + MASK_WORDS * self.experts
@property
def expert_slot0(self) -> int:
"""The route table's per-expert slots (a slot where the expert is in U)."""
return 3 * self.picks + 1 + self.max_u
@property
def order0(self) -> int:
"""The picks (t topk + k) in slot order, a slot's ascending."""
return self.expert_slot0 + self.experts
@property
def first0(self) -> int:
"""Each slot's first position in the pick order, then the pick count."""
return self.order0 + self.picks
@property
def mask0(self) -> int:
"""Each expert's tokens that picked it, a bit a token (``MASK_WORDS``)."""
return self.first0 + self.max_u + 1
@property
def tile_picks(self) -> int:
"""A slot's picks at most: a token picks an expert once."""
return min(self.tokens, 16)
def route_w(self, tab, s, t, k):
return fx.ptr_load(tab + (t * 2 * self.topk + self.topk + k)).bitcast(
fx.Float32
)
def n_union(self, tab, s):
return fx.ptr_load(tab + 2 * s * self.topk)
def union_expert(self, tab, s, u):
return fx.ptr_load(tab + (2 * s * self.topk + 1 + u))
def slot_first(self, tab, u):
return fx.ptr_load(tab + (self.first0 + u))
def pick_at(self, tab, i):
return fx.ptr_load(tab + (self.order0 + i))
def tile_first(self, tab, u):
return fx.ptr_load(tab + (self.tile_first0 + u))
def tile_slot(self, tab, i):
return fx.ptr_load(tab + (self.tile_slot0 + i))
def route_shape(key: MoeBuild) -> RouteShape:
return RouteShape(key.experts, key.topk, key.tokens, key.tp, key.ug_groups)
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/model_ops/deepseek_v41/router.py
"""V4.1's router GEMV tile for decode: bf16 x [T, 5120] times the gate weight
[384, 5120] -> f32 logits, in one fixed summation order.
Rows 16 a tile, K in ``PARTS`` contiguous parts, 16x16x32 bf16 MFMA: in part
p wave w takes the part's K steps w, w + 8, ... (32 columns each) in that
order, and the eight wave partials are summed in wave order
(``common.ops.row_sum``); the MoE's route stage sums the parts in part
order. The MoE stages spread a tile's parts over PARTS CTAs, and past 16 tokens
a token tile a CTA too, which the order allows: a CTA alone cannot read a
tile, or every token's x, fast enough.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, range_constexpr
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import mfma_bf16, rsrc
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
HIDDEN = 5120
ROWS = 16 # a tile's rows: the MFMA's M
PARTS = 4 # contiguous K parts of a tile, summed in order
def _k0s(part, lane, wave):
"""The K offsets of this lane's loads in part ``part`` (8 bf16 each)."""
per_wave = HIDDEN // 32 // WAVES // PARTS
assert per_wave * 32 * WAVES * PARTS == HIDDEN
return [
(part * per_wave * WAVES + wave + WAVES * i) * 32 + 8 * (lane // 16)
for i in range(per_wave)
]
def router_weight_loads(w, task, part, lane, wave, w_cm=0):
"""``router_tile``'s weight operands (rows 16 task .., K part ``part``).
``w_cm``: their load cache policy (nontemporal where nothing else of the
step reuses them)."""
row = task * ROWS + lane % 16
return [
fx.Vector(
bo.buffer_load(
rsrc(w),
(row * HIDDEN + k0) // 2,
vec_width=4,
dtype=T.i32,
cache_modifier=w_cm,
)
)
for k0 in _k0s(part, lane, wave)
]
def router_x_loads(x, part, s, lane, wave, x_cm=0, t0=0):
"""``router_tile``'s x operands: the s tokens from ``t0`` (column
lane % 16; ``t0`` traced or an int). ``x_cm``: their load cache policy
(device scope when another CTA of the same launch wrote x)."""
tok = fx.min(lane % 16, s - 1)
if const_expr(not isinstance(t0, int) or t0 != 0):
tok = tok + t0
return [
fx.Vector(
bo.buffer_load(
rsrc(x),
(tok * HIDDEN + k0) // 2,
vec_width=4,
dtype=T.i32,
cache_modifier=x_cm,
)
)
for k0 in _k0s(part, lane, wave)
]
def router_tile_mfma(wvs, xvs, lane, wave, red):
"""``router_tile``'s MFMAs, K steps in order -> ``red[wave][lane]``."""
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i in range_constexpr(len(wvs)):
acc = mfma_bf16(wvs[i].bitcast(fx.BFloat16), xvs[i].bitcast(fx.BFloat16), acc)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (slice, gate, norm) and
# atom/models/deepseek_v41/mono/kernels/attn_post.py (the attention's TP reduce).
"""The mono layer's mHC seam stages, each a task table of a persistent launch
handing off through the tagged mailbox:
slice (160, 32 hidden columns x 4 streams each): the owed post folded into
the residual (R_new, bf16 out), the collapsed layer input (mailbox
LIN) and the partial mix projection / sum of squares (mailbox PMIX)
gate (S): the 160 partials -> post / comb (Sinkhorn) and the next pre
norm (S): the RMSNorm -> the bf16 row, handed to the caller's store
reduce (160): the FFN seam's attention output, the TP ranks' partials
(pushed to every rank's ATTN region) summed in the all-reduce's
order -> ``pend_lds``, where its slice reads it
push (160): a rank's partial given in memory (an attention vLLM ran)
to every rank's ATTN region, ahead of the slice's reduce
Every elementwise step follows aiter's ``mhc_fused_post_pre_delayed_rmsnorm``
rounding; the mix projection and the sums of squares, split-K reductions,
differ only in their split and order.
"""
import math
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import (
batched_rounds,
bf16_round,
bf_hi,
bf_lo,
butterfly,
hw_exp2,
hw_rcp,
hw_rsq,
lane_gather,
rsrc,
sum_partials,
traced,
)
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES, pair_layout
from vllm.models.deepseek_v41.amd.mono.common.sync import preg
from vllm.models.deepseek_v41.amd.mono.stages.gemv import (
ld_bf,
ld_f32,
plus,
token_tiles,
)
HC = 4
HIDDEN = 5120
K_MIX = HC * HIDDEN
MIX = HC * (HC + 2) # 24 projections: pre 4 | post 4 | comb 16
EPS = 1e-20
HC_EPS = 1e-6
SINKHORN = 20
POST_MULT = 2.0
LOG2E = 1.0 / math.log(2.0)
COLS = 32 # hidden columns a slice task
SLICES = HIDDEN // COLS # 160
KT = HC * COLS # projection columns a slice task
PART = MIX + 1 # a slice's partials a token: the mixes, then the sum of squares
GATE_BLOCK = 8 # slices a gate thread sums
def scratch_layout(tokens: int) -> dict[str, tuple[int, int]]:
"""Mailbox region -> (byte offset, bytes) of a seam; 8 B a pair."""
pairs = {
"lin": tokens * HIDDEN // 2, # bf16 pairs
"pmix": SLICES * tokens * PART,
}
return pair_layout(pairs.items())
def attn_peer_bytes(tokens: int, tp: int) -> int:
"""ATTN: ``[source rank][token][hidden / 2]`` bf16 pairs."""
return tp * tokens * HIDDEN // 2 * 8
def attn_region_pair(src, t, s, col):
return (src * s + t) * (HIDDEN // 2) + col // 2
def fma(a, b, c):
return fx.Float32(fmath.fma(a, b, c))
def sigmoid(z):
"""Triton's ``tl.sigmoid``: 1 / (1 + exp2(-z log2(e)))."""
return fx.Float32(1.0) / (fx.Float32(1.0) + hw_exp2(z * fx.Float32(-LOG2E)))
def lane_f32(v, src):
return lane_gather(v.bitcast(fx.Int32), src).bitcast(fx.Float32)
def pend_value(c, t, col, cc):
"""The owed post's sublayer output at (token t, column col): read from
``pend``, or where a caller staged it (``pend_lds``, ``[token][32]`` of this
task's columns: a reduction it did first)."""
if const_expr(c.get("pend_lds") is None):
return ld_bf(c["args"]["pend"], t * HIDDEN + col)
return fx.ptr_load(c["pend_lds"] + (t * COLS + cc))
# ---------------------------------------------------------------- slice
@traced
def slice_input(c, task, idx):
"""Thread ``idx``'s pair of the layer input (token idx / 16, columns
32 task + 2 (idx % 16) ..)."""
s, rl = c["S"], c["rl"]
a = c["args"]
c0 = task * COLS
if idx < s * COLS // 2:
t = idx // (COLS // 2)
cc = 2 * (idx % (COLS // 2))
outs = []
for d in range_constexpr(2):
acc = fx.Float32(0.0)
for h in range_constexpr(HC):
rb = fx.ptr_load(rl + (t * KT + h * COLS + cc + d))
acc = acc + ld_f32(a["pre_in"], t * HC + h) * rb
outs.append(acc)
c["put_bf"](c["lin"], t * HIDDEN + c0 + cc, outs)
def slice_at(c0, e):
"""Element e of a slice task at column c0: (token, stream, hidden column)."""
return e // KT, (e % KT) // COLS, c0 + e % COLS
def slice_loads(c, c0, e):
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
a = c["args"]
t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
return r, cm, ld_f32(a["post_in"], t * HC + o)
def slice_row(c, c0, e, got):
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
a = c["args"]
t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
# a settled residual (no owed post): stream o as it arrives
v = got[0]
else:
r, cm, post = got
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
for h in range_constexpr(1, HC - 1):
v = v + cm[h] * r[h]
# bf16 R_new is what the collapse, the sums of squares and the
# mix projection all read
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
@traced
def stage_slice(c, task):
"""R_new, the layer input and the partial projections of hidden columns
32 task .. +32, all four streams, every token."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
rl, fl = c["rl"], c["fl"]
a = c["args"]
c0 = task * COLS
batched_rounds(
tid,
s * KT,
lambda e: slice_loads(c, c0, e),
lambda e, got: slice_row(c, c0, e, got),
)
for i in range_constexpr((MIX * KT + THREADS - 1) // THREADS):
e = tid + THREADS * i
if e < MIX * KT:
j = e // KT
o = (e % KT) // COLS
# the original's two bf16 MFMAs take fn as hi = bf16(fn) and lo =
# bf16(fn - hi): their sum, exact in fp32, is the weight it applies
f = ld_f32(a["hc_fn"], j * K_MIX + o * HIDDEN + c0 + e % COLS)
hi = bf16_round(f)
fx.ptr_store(hi + bf16_round(f - hi), fl + e)
gpu.barrier()
# the layer input: bf16 R_new of the four streams by the incoming pre-mix,
# products rounded, summed in order (the Triton collapse, no contraction);
# two columns a thread, one bf16 pair a mailbox store
for pas in range_constexpr(0, s * COLS // 2, THREADS):
slice_input(c, task, plus(pas, tid))
# sum of squares: token t on wave t % WAVES, two columns a lane
for t in range_constexpr(s):
if wave == t % WAVES:
x0 = fx.ptr_load(rl + (t * KT + 2 * lane))
x1 = fx.ptr_load(rl + (t * KT + 2 * lane + 1))
sq = butterfly(x0 * x0 + x1 * x1, (32, 16, 8, 4, 2, 1))
if lane == 0:
c["put"](c["pmix"], (task * s + t) * PART + MIX, sq)
# mixes: fp32 MFMA 16x16x4, A = fn rows (two 16-row tiles), B = R_new^T,
# a token tile at a time
for t0, n in token_tiles(s):
slice_mix(c, task, t0, n)
@traced
def slice_mix(c, task, t0, n):
"""The mix projection partials of tokens t0 .. t0 + n (``stage_slice``)."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
rl, fl, red = c["rl"], c["fl"], c["red"]
tok = plus(t0, fx.min(lane % 16, n - 1))
kw = KT // WAVES
cs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(2)]
for kk in range_constexpr(kw // 4):
colk = wave * kw + lane // 16 + 4 * kk
b = fx.ptr_load(rl + (tok * KT + colk))
for h in range_constexpr(2):
row = h * 16 + lane % 16
av = fx.ptr_load(fl + (fx.min(row, MIX - 1) * KT + colk))
av = (row < MIX).select(av, fx.Float32(0.0))
ops = rocdl._split_mfma_operands([av, b, cs[h], 0, 0, 0])
cs[h] = fx.Vector(rocdl.mfma_f32_16x16x4f32(T.vec(4, T.f32), *ops).result)
for h in range_constexpr(2):
fx.ptr_store(cs[h], red + ((wave * 2 + h) * 64 + lane) * 4)
gpu.barrier()
if tid < MIX * n:
tl = tid // MIX
t = plus(t0, tl)
j = tid % MIX
h = j // 16
ln = 16 * ((j % 16) // 4) + tl
tot = fx.Float32(0.0)
for w in range_constexpr(WAVES):
tot = tot + fx.ptr_load(red + (((w * 2 + h) * 64 + ln) * 4 + j % 4))
c["put"](c["pmix"], (task * s + t) * PART + j, tot)
gpu.barrier()
# ---------------------------------------------------------------- gate
def _row(v, lane, q):
"""Comb lanes 8..23 hold ``comb[r][c]`` at lane 8 + 4 r + c: element q of
the row."""
return lane_f32(v, (lane - 8) // 4 * 4 + 8 + q)
def _col(v, lane, q):
return lane_f32(v, 8 + 4 * q + (lane - 8) % 4)
@traced
def stage_gate(c, t):
"""Token t's 160 slices, in slice order, -> post / comb / next pre."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
red = c["red"]
a = c["args"]
groups = SLICES // GATE_BLOCK # 20
if tid < groups * PART:
g = tid // PART
j = tid % PART
vals = c["poll"](
[
(c["pmix"], ((g * GATE_BLOCK + k) * s + t) * PART + j, 1)
for k in range(GATE_BLOCK)
]
)
acc = vals[0][0].bitcast(fx.Float32)
for k in range_constexpr(1, GATE_BLOCK):
acc = acc + vals[k][0].bitcast(fx.Float32)
fx.ptr_store(acc, red + (g * PART + j))
gpu.barrier()
if wave == 0:
j = fx.min(lane, MIX - 1)
mix = fx.ptr_load(red + j)
sq = fx.ptr_load(red + MIX)
for g in range_constexpr(1, groups):
mix = mix + fx.ptr_load(red + (g * PART + j))
sq = sq + fx.ptr_load(red + (g * PART + MIX))
sc = [ld_f32(a["hc_scale"], i) for i in range(3)]
base = ld_f32(a["hc_base"], j)
# aiter's mhc_fused_post_pre_delayed_rmsnorm reduce: one rstd =
# rsq(fma(sum, 1/K, eps)), then each gate fma(mix rstd, scale, base)
v = mix * hw_rsq(fma(sq, fx.Float32(1.0 / K_MIX), fx.Float32(EPS)))
if lane < HC:
pre = sigmoid(fma(v, sc[0], base))
bo.buffer_store(pre + fx.Float32(HC_EPS), rsrc(a["pre_out"]), t * HC + lane)
if (lane >= HC) & (lane < 2 * HC):
post = sigmoid(fma(v, sc[1], base))
bo.buffer_store(
post * fx.Float32(POST_MULT), rsrc(a["post_out"]), t * HC + lane - HC
)
cv = fma(v, sc[2], base)
row = [_row(cv, lane, q) for q in range(4)]
# Triton's exp: exp2 of (x - max) log2(e)
m = fx.max(fx.max(row[0], row[1]), fx.max(row[2], row[3]))
cv = hw_exp2((cv - m) * fx.Float32(LOG2E))
# fast_dividef is x rcp(y); its first "+ eps" contracts into the fma
row = [_row(cv, lane, q) for q in range(4)]
cv = fma(cv, hw_rcp((row[0] + row[1]) + (row[2] + row[3])), fx.Float32(HC_EPS))
col = [_col(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS))
for _ in range_constexpr(SINKHORN - 1):
row = [_row(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(
((row[0] + row[1]) + (row[2] + row[3])) + fx.Float32(HC_EPS)
)
col = [_col(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(
((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS)
)
if (lane >= 2 * HC) & (lane < MIX):
bo.buffer_store(cv, rsrc(a["comb_out"]), t * HC * HC + lane - 2 * HC)
gpu.barrier()
# ---------------------------------------------------------------- norm
@traced
def stage_norm(c, t, store):
"""Token t's RMSNorm (rstd = rsq(fma(sum, 1/H, eps)), bf16((x rstd) w));
256 threads, 3 chunks of 8 at 8 t + 2048 c, each chunk's bf16 values handed
to ``store(t, col, ys, live)`` by every thread (``live``: a thread's chunk
in the row; threads 256.. mirror thread 255)."""
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
a = c["args"]
tt = fx.min(tid, 255)
mine = tid < 256
cols = [tt * 8 + 2048 * ch for ch in range(3)]
xs = []
for ch in range_constexpr(3):
base = t * HIDDEN + fx.min(cols[ch], HIDDEN - 8)
words = c["poll"]([(c["lin"], base // 2 + k, 1) for k in range(4)])
v = []
for k in range_constexpr(4):
v += [bf_lo(words[k][0]), bf_hi(words[k][0])]
xs.append([(cols[ch] < HIDDEN).select(x, fx.Float32(0.0)) for x in v])
acc = fx.Float32(0.0)
for ch in range_constexpr(3):
for x in xs[ch]:
acc = acc + x * x
acc = butterfly(acc, (1, 2, 4, 8, 16, 32))
if (lane == 0) & mine:
fx.ptr_store(acc, red + wave)
gpu.barrier()
tot = butterfly(fx.ptr_load(red + lane % 4), (1, 2))
r = hw_rsq(fma(tot, fx.Float32(1.0 / HIDDEN), fx.Float32(EPS)))
for ch in range_constexpr(3):
col = fx.min(cols[ch], HIDDEN - 8)
ys = [
bf16_round((xs[ch][j] * r) * ld_bf(a["norm_w"], col + j)) for j in range(8)
]
store(t, col, ys, mine & (cols[ch] < HIDDEN))
gpu.barrier()
# ---------------------------------------------------------------- reduce
@traced
def stage_reduce(c, task):
"""The attention output at this slice's 32 columns: the TP ranks' partials
in the all-reduce's order (``sum_partials``), in fp32 -> bf16 ->
``pend_lds``."""
s, tid = c["S"], c["tid"]
for pas in range_constexpr(0, s * COLS // 2, THREADS):
reduce_attn_pair(c, task, plus(pas, tid))
gpu.barrier()
@traced
def reduce_attn_pair(c, task, idx):
"""Thread ``idx``'s column pair of token idx / 16 (``stage_reduce``)."""
s = c["S"]
c0 = task * COLS
if idx < s * COLS // 2:
t = idx // (COLS // 2)
col = c0 + 2 * (idx % (COLS // 2))
own = preg(c["sym"], 0, "attn")
acc0, acc1 = sum_partials(
c["poll"],
own,
lambda src: attn_region_pair(src, t, s, col),
c["d"].tp,
)
lds = c["pend_lds"] + (t * COLS + col - c0)
fx.ptr_store(bf16_round(acc0), lds)
fx.ptr_store(bf16_round(acc1), lds + 1)
@traced
def stage_push(c, task, part):
"""This rank's attention partial (``part``: bf16 [S, HIDDEN], wo_b's
unreduced output) at this slice's 32 columns, every token, to every rank's
ATTN region: the pairs ``stage_reduce`` polls, as the bits vLLM wrote."""
s, tid, tp = c["S"], c["tid"], c["d"].tp
for pas in range_constexpr(0, s * COLS // 2, THREADS):
idx = plus(pas, tid)
if idx < s * COLS // 2:
t = idx // (COLS // 2)
col = task * COLS + 2 * (idx % (COLS // 2))
w = bo.buffer_load(
rsrc(part), (t * HIDDEN + col) // 2, vec_width=1, dtype=T.i32
)
pair = attn_region_pair(c["rank"], t, s, col)
for p in range_constexpr(tp):
c["put_words"](preg(c["peer_addr"](p), 0, "attn"), pair, [w])
+435
View File
@@ -0,0 +1,435 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
A decode step's backbone layer -- the attention seam, the attention, its TP
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
FlyDSL launches (``mono``, this directory) on the layer's loaded
weights, the TP reductions inside the kernels. Every backbone layer of the
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
the first layer and the Engram layers hand K1 a settled residual, and an index
source runs vLLM's compressor and indexer between the two launches. A layer
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
and attention and runs the rest as one launch, the FFN launch, from wo_b's
unreduced output. A step takes the kernels when it is decode only, at most
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
a whole layer also needs causal SWA windows. Anything else runs the layer as
before; the paths share every tensor at the layer boundary, so they interleave
freely.
Every TP rank must take the same path at every layer (the kernels wait on each
other's pushes): the conditions read only metadata every rank holds alike.
"""
from typing import TYPE_CHECKING, Any, cast
import torch
import vllm.envs as envs
from vllm.config import CUDAGraphMode
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
from vllm.models.deepseek_v41.amd.rocm import DeepseekV4ROCMAiterSparseSWAMetadata
from vllm.models.deepseek_v41.sparse_mla import DeepseekV4FlashMLAMetadata
logger = init_logger(__name__)
RECORD = 584 # an fp8_ds_mla KV record: 576 data bytes, 8 scale bytes
SWA_WIDTH = 128 # a causal decode token's window slots
MAX_ROWS = 48 # the kernels' step rows at most: 8 requests x (1 + 5 DSpark drafts)
# the kernels' ISA: CDNA4 (gfx950) scaled MFMA and MX formats
CDNA_VERSIONS = (4,)
_runner = None
def _mono_runner(device: torch.device):
"""The process's mono runner (one a TP rank, shared by every layer). Its
peer-memory handshake is collective over the TP group: every rank builds it
at the same layer of the same step."""
global _runner
if _runner is None:
if torch.cuda.is_current_stream_capturing():
# its peer memory is allocated and exchanged eagerly; vLLM warms
# every graph up eagerly before capturing it
raise RuntimeError(
"--monokernel was first reached inside a CUDA graph "
"capture, before an eager decode step."
)
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
assert MAX_TOKENS >= MAX_ROWS, MAX_TOKENS
_runner = DSV41MonoLayer(
get_tensor_model_parallel_world_size(),
get_tensor_model_parallel_rank(),
get_tp_group().cpu_group,
device,
)
return _runner
def _record_view(kv: torch.Tensor, block: int) -> torch.Tensor:
"""A layer's fp8_ds_mla cache as [blocks, block, RECORD] bytes (a block's
data rows, then its scale words: the kernels index both from it)."""
kv = kv if kv.dtype == torch.uint8 else kv.view(torch.uint8)
assert kv.stride(0) >= block * RECORD, (tuple(kv.shape), kv.stride(), block)
return torch.as_strided(kv, (kv.shape[0], block, RECORD), (kv.stride(0), RECORD, 1))
def _scale_bytes(scale: torch.Tensor) -> torch.Tensor:
return scale if scale.dtype == torch.uint8 else scale.view(torch.uint8)
def _compute_units() -> int:
return current_platform.num_compute_units(torch.accelerator.current_device_index())
def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None:
"""Why the kernels cannot run this deployment's MoE, or None. They read every
one of the 384 experts on each rank, TP-sharded, in AITER's A8W4 MXFP4
layout."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
pc = vllm_config.parallel_config
if pc.enable_expert_parallel or pc.enable_eplb or pc.data_parallel_size > 1:
return "needs tensor parallelism only: no expert or data parallelism"
if envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
return "needs AITER's A8W4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 unset)"
experts = getattr(getattr(layer.ffn, "experts", None), "routed_experts", None)
if experts is None:
return None # not a routed MoE: the layer checks decline it
backend = getattr(experts.quant_method, "mxfp4_backend", None)
if backend != Mxfp4MoeBackend.AITER_MXFP4_BF16:
return f"needs the AITER_MXFP4_BF16 MoE backend, not {backend}"
return None
def _select(
layer: "DeepseekV4DecoderLayer",
positions: torch.Tensor,
normed: torch.Tensor,
qr: torch.Tensor,
) -> None:
"""An index source's compressor and top-k selection, from K1's normed input
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
indexer op, serial."""
attn = layer.attn
comp, indexer = attn.compressor, attn.indexer
assert indexer is not None
if comp is not None:
kv_score = torch.mm(
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
)
latent = comp(kv_score, positions)
comp.insert_cache(latent, positions, attn.rotary_emb)
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
weights, _ = indexer.weights_proj(normed)
q, q_scale, weights = indexer.forward_q(
qr, None, weights, positions, attn.indexer_rotary_emb
)
assert weights is not None
indexer.indexer_op(
normed, (q, q_scale) if q_scale is not None else q, None, weights
)
class MonoDecodeLayer:
"""One decoder layer's mono path (``create``: None when the layer is not
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
launch after vLLM's attention (``ffn``). Its weights are taken from the
layer at the first call, after loading."""
def __init__(self, ffn_only: bool = False) -> None:
self.ffn_only = ffn_only
self._weights: MonoLayerWeights | None = None
@staticmethod
def create(
layer: "DeepseekV4DecoderLayer", vllm_config
) -> "MonoDecodeLayer | None":
if not vllm_config.kernel_config.monokernel:
return None
from vllm.platforms.rocm import get_cdna_version
# the deployment: an explicit opt-in that cannot run is an error
why = None
if get_cdna_version() not in CDNA_VERSIONS:
why = f"needs CDNA{'/'.join(map(str, CDNA_VERSIONS))}"
elif get_tensor_model_parallel_world_size() not in (2, 4):
why = "needs tensor parallel size 2 or 4"
else:
why = _unsupported_moe(layer, vllm_config)
if why is None:
try:
from vllm.models.deepseek_v41.amd.mono.runner import BLOCKS
except ImportError as err:
why = f"needs FlyDSL and AITER's FlyDSL helpers ({err})"
else:
# every CTA of a launch stays resident: a partitioned GPU deadlocks
cus = _compute_units()
if cus < BLOCKS:
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
if why is not None:
raise ValueError(f"--monokernel {why}.")
# the layer: the kernels serve the backbone's seams and MoE
config = vllm_config.model_config.hf_config
attn, ffn = layer.attn, layer.ffn
if layer.use_sequence_parallel:
why = "sequence-parallel layer"
elif not layer.fuse_seam_norm:
why = "needs aiter's fused mHC seam"
elif attn.layer_id >= config.num_hidden_layers:
why = "draft layer"
elif (
ffn.shared_experts is None
or ffn.gate.tid2eid is not None
or ffn.n_routed_experts != 384
or ffn.n_activated_experts != 6
or ffn.routed_scaling_factor != 1.5
or ffn.swiglu_limit != 10.0
or ffn.scoring_func != "sqrtsoftplus"
or not ffn.renormalize
):
why = "MoE routing outside the kernels' (384 / 6, sqrtsoftplus, x1.5)"
if why is not None:
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
return None
# the whole layer: the kernels' attention takes the standard layers; an
# index source's compressor and indexer run between its two launches
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
if attn.compressor is not None and attn.indexer is None:
why = "a compressor without an indexer"
elif attn.indexer is not None and not isinstance(
attn.indexer, DeepseekV41RocmMxfp4Indexer
):
why = f"a {type(attn.indexer).__name__}"
elif attn.compress_ratio not in (0, 1, 2):
why = f"compress ratio {attn.compress_ratio}"
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
why = f"a {attn.kv_cache_dtype} KV cache"
if why is not None:
logger.info_once(
"DSv4.1 mono decode: the FFN launch for layers with %s", why
)
return MonoDecodeLayer(ffn_only=True)
logger.info_once("DSv4.1 mono decode: the whole standard backbone layers")
return MonoDecodeLayer()
def weights(self, layer: "DeepseekV4DecoderLayer"):
if self._weights is not None:
return self._weights
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
a, f = layer.attn, layer.ffn
e, sh = f.experts.routed_experts, f.shared_experts
assert sh is not None # ``create`` took only layers with one
self._weights = MonoLayerWeights(
attn=None if self.ffn_only else self._attn_weights(a),
hc_attn_fn=layer.hc_attn_fn,
hc_attn_scale=layer.hc_attn_scale,
hc_attn_base=layer.hc_attn_base,
attn_norm=layer.attn_norm.weight,
hc_ffn_fn=layer.hc_ffn_fn,
hc_ffn_scale=layer.hc_ffn_scale,
hc_ffn_base=layer.hc_ffn_base,
ffn_norm=layer.ffn_norm.weight,
gate_w=f.gate.weight,
bias=f.gate.e_score_correction_bias,
w13=e.w13_weight,
w13_s=e.w13_weight_scale,
w2=e.w2_weight,
w2_s=e.w2_weight_scale,
sgu=sh.gate_up_proj.weight,
sgu_s=_scale_bytes(sh.gate_up_proj.weight_scale),
sw2=sh.down_proj.weight,
sw2_s=_scale_bytes(sh.down_proj.weight_scale),
)
self._weights.check(get_tensor_model_parallel_world_size())
return self._weights
@staticmethod
def _attn_weights(a):
from vllm.models.deepseek_v41.amd.mono.attention.plan import Dims
from vllm.models.deepseek_v41.amd.mono.runner import AttnWeights
attn = AttnWeights(
layer_id=a.layer_id,
wqkv=a.fused_wqa_wkv.weight,
wqkv_scale=_scale_bytes(a.fused_wqa_wkv.weight_scale),
q_norm=a.q_norm.weight,
kv_norm=a.kv_norm.weight,
wq_b=a.wq_b.weight,
wq_b_scale=_scale_bytes(a.wq_b.weight_scale),
wo_a=a.wo_a.weight,
wo_a_scale=_scale_bytes(a.wo_a.weight_scale),
wo_b=a.wo_b.weight,
wo_b_scale=_scale_bytes(a.wo_b.weight_scale),
attn_sink=a.attn_sink,
cos_sin=a.rotary_emb.cos_sin_cache,
ratio=a.compress_ratio,
)
attn.check(Dims(get_tensor_model_parallel_world_size()))
return attn
@staticmethod
def _skip(why: str) -> None:
logger.debug_once("DSv4.1 mono decode skipped a step: %s", why)
return None
@staticmethod
def _decode_step(layer: "DeepseekV4DecoderLayer", rows: int):
"""(why this step takes the original path -- None for a decode-only step
the kernels take -- the layer's SWA metadata, its compressed cache's)."""
if not 1 <= rows <= MAX_ROWS:
return f"{rows} rows", None, None
if not is_forward_context_available():
return "no forward context", None, None
fc = get_forward_context()
md = fc.attn_metadata
attn = layer.attn
swa, comp = None, None
if isinstance(md, dict):
swa = cast(
"DeepseekV4ROCMAiterSparseSWAMetadata | None",
md.get(attn.swa_cache_layer.prefix),
)
if attn.compressed_cache_prefix is not None:
comp = cast(
"DeepseekV4FlashMLAMetadata | None",
md.get(attn.compressed_cache_prefix),
)
why = None
if fc.cudagraph_runtime_mode not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL):
# a PIECEWISE (breakable) capture is replayed for mixed batches too
why = f"{fc.cudagraph_runtime_mode} step"
elif not isinstance(md, dict): # a profile / dummy run, or DBO ubatches
why = "no attention metadata"
elif swa is None:
why = "layer metadata missing"
elif swa.num_prefills != 0 or swa.num_decodes == 0:
why = "not a decode-only step"
elif swa.num_decode_tokens > rows:
why = "more decode tokens than rows"
return why, swa, comp
def __call__(
self,
layer: "DeepseekV4DecoderLayer",
x: torch.Tensor,
positions: torch.Tensor,
residual: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
pre_mix: torch.Tensor | None,
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
mono kernels, or None when this step takes the original path (always for
an ``ffn_only`` layer)."""
if self.ffn_only:
return None
first = residual is None
if first and x.dim() != 2:
return self._skip("first layer of a pipeline stage")
why, swa, comp = self._decode_step(layer, x.shape[0])
attn = layer.attn
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
if why is None:
# the kernels' attention: causal SWA windows, the compressed cache
if comp is None and attn.compress_ratio:
why = "layer metadata missing"
elif swa.decode_swa_width != SWA_WIDTH:
why = f"SWA width {swa.decode_swa_width}"
elif (
swa.decode_swa_indices is None
or swa.decode_swa_lens is None
or swa.token_to_req_indices is None
):
why = "SWA decode metadata missing"
elif not all(t is not None and t.is_contiguous() for t in tensors):
why = "non-contiguous layer inputs"
if why is not None:
return self._skip(why)
assert swa is not None
runner = _mono_runner(x.device)
logger.info_once(
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
)
# K1's seam folds the owed post into the residual, except where the
# residual arrives settled: the embedding, or after the Engram write
fold = True
if first:
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
pre_mix = runner.first_pre_mix(x.shape[0])
post_mix = res_mix = pre_mix # unread without the fold
fold = False
elif layer.engram is not None and engram_hashes is not None:
residual = layer.mhc_post(x, residual, post_mix, res_mix)
hashes = engram_hashes[:, layer.engram.layer_hash_index]
residual = layer.engram(residual, hashes, engram_mask)
fold = False
kw: dict[str, Any] = {}
if attn.indexer is not None:
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
if attn.compress_ratio:
assert comp is not None
kw.update(
topk_indices=attn.topk_indices_buffer,
comp_cache=_record_view(
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
)
return runner.forward(
self.weights(layer),
x,
residual,
post_mix,
res_mix,
pre_mix,
positions,
swa.slot_mapping,
_record_view(attn.swa_cache_layer.kv_cache, swa.block_size),
swa.decode_swa_indices,
swa.decode_swa_lens,
swa.token_to_req_indices,
fold=fold,
**kw,
)
def ffn(
self,
layer: "DeepseekV4DecoderLayer",
part: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor,
res_mix: torch.Tensor,
pre_mix: torch.Tensor,
) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs by the FFN launch from ``part``, wo_b's unreduced
output, and the attention seam's outputs; None when this step takes the
original path (the caller then reduces ``part`` itself)."""
why, _, _ = self._decode_step(layer, part.shape[0])
tensors = (part, residual, post_mix, res_mix, pre_mix)
if why is None and not all(t.is_contiguous() for t in tensors):
why = "non-contiguous layer inputs"
if why is not None:
return self._skip(why)
runner = _mono_runner(part.device)
logger.info_once(
"DSv4.1 mono decode FFN launch active (decode steps of <= %d rows)",
MAX_ROWS,
)
return runner.ffn(self.weights(layer), *tensors)
+36 -3
View File
@@ -8,6 +8,7 @@ from typing import Any, cast
import torch
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
@@ -41,6 +42,7 @@ from vllm.platforms.rocm import _ON_GFX950
from vllm.triton_utils import tl, triton
from vllm.utils.multi_stream_utils import execute_in_parallel
from vllm.v1.attention.backend import (
AttentionCGSupport,
CommonAttentionMetadata,
MultipleOf,
)
@@ -61,6 +63,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
rocm_paged_mxfp4_cache_layout,
)
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.worker.workspace import current_workspace_manager
logger = init_logger(__name__)
@@ -285,7 +288,35 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
prefill_swa_ragged_indptr: torch.Tensor | None = None
class DeepseekV41ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.enable_adaptive_verification:
# Per-token request ownership and compressed slot mappings are built
# from device query boundaries into persistent buffers.
return AttentionCGSupport.ALWAYS
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBuilder):
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.enable_adaptive_verification:
# SWA indices, lengths, and request ownership are built from device
# query boundaries and copied into persistent ragged buffers.
return AttentionCGSupport.ALWAYS
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
# also refreshes the ROCm-specific ragged SWA indices and indptrs.
supports_draft_decode_metadata_update = False
@@ -508,8 +539,8 @@ class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend):
return [64, 128]
@staticmethod
def get_builder_cls() -> type[DeepseekV4SparseMLAMetadataBuilder]:
return DeepseekV4SparseMLAMetadataBuilder
def get_builder_cls() -> type[DeepseekV41ROCMAiterMLASparseMetadataBuilder]:
return DeepseekV41ROCMAiterMLASparseMetadataBuilder
class DeepseekV41ROCMAiterSparseSWABackend(DeepseekSparseSWABackend):
@@ -929,7 +960,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
def _wo_b_after_wo_a(self, zf: torch.Tensor) -> torch.Tensor:
if self._wo_b_scale is not None and zf.dim() == 2:
return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True)
return self._bpre_attn_gemm(
self.wo_b.weight, self._wo_b_scale, zf, self.wo_b.reduce_results
)
return self.wo_b(zf)
def forward_mqa(
+5
View File
@@ -51,6 +51,7 @@ from ..common.mm_preprocess import (
DeepseekV4VLProcessingInfo,
)
from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin
from ..decoder_replay_layers import DecoderReplayLayers
from .model import (
DeepseekV41LLMForCausalLM,
_linear_scale_param_name,
@@ -299,6 +300,10 @@ class DeepseekV41ForCausalLM(
def token_lookback_depth(self) -> int:
return self.language_model.token_lookback_depth
@property
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
return self.language_model.decoder_replay_layers
def forward(
self,
input_ids: torch.Tensor,
+7 -9
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
from vllm.models.deepseek_v41.common.ops import (
@@ -356,10 +357,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
# Sink padded to the same head count, initialized to -inf (no sink
# effect). Weight loading fills the first n_local_heads slots.
self.attn_sink = nn.Parameter(
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
requires_grad=False,
)
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
self.fused_wqa_wkv = MergedColumnParallelLinear(
self.hidden_size,
@@ -547,11 +545,11 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
swa_bounded_replay = False
if swa_bounded_replay and current_platform.is_rocm():
logger.warning_once(
"SWA bounded replay is off on ROCm (the sparse SWA metadata "
"builders forward replay_start, but the window clamp it relies "
"on lives in the FlashInfer and FlashMLA prefill kernels, so "
"the padded slots fault); the sliding-window cache takes part "
"in prefix caching instead."
"Prefix-cache SWA replay is off on ROCm (the sparse SWA "
"metadata builders forward replay_start, but the window clamp "
"it relies on lives in the FlashInfer and FlashMLA prefill "
"kernels, so the padded slots fault); the sliding-window cache "
"takes part in prefix caching instead."
)
swa_bounded_replay = False
self.swa_cache_layer = DeepseekV4SWACache(
@@ -31,6 +31,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonWarmupTensor,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import cdiv, next_power_of_2
@@ -1012,7 +1013,7 @@ def _combine_topk_swa_warmup_inputs(vllm_config: Any) -> _WarmupInputRows:
"""
model_config = getattr(vllm_config, "model_config", None)
hf_config = getattr(model_config, "hf_config", None)
compress_ratios = getattr(hf_config, "compress_ratios", None) or [0]
compress_ratios = get_compress_ratios(hf_config) or [0]
topk = _hf_config_int(vllm_config, "index_topk", 0)
return zip_inputs(
*(
@@ -17,6 +17,8 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.ops.common import pack_ue8m0_x4
from . import MXFP4_BLOCK_SIZE, _fp32x2_to_fp4x2
@@ -51,7 +53,8 @@ def indexer_k_norm_rope_store(
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
compress_ratio: group size; keys are emitted at group boundaries.
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
per-token FP8 with a single fp32 scale.
per-token FP8 with a single fp32 scale (four packed ue8m0 bytes
under `mqa_logits_use_mx_scales`).
"""
num_tokens = kv_slot_mapping.numel()
@@ -111,6 +114,7 @@ def indexer_k_norm_rope_store(
KV_BLOCK_STRIDE=k_cache.stride(0),
FP8_MAX=448.0,
USE_FP4=use_fp4_cache,
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
SHUFFLE=shuffle,
BLOCK_TILE_SIZE=_BLOCK_TILE_SIZE,
HEAD_TILE_SIZE=_HEAD_TILE_SIZE,
@@ -139,6 +143,7 @@ def _indexer_k_norm_rope_quant_store_kernel(
KV_BLOCK_STRIDE: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FP4: tl.constexpr,
MX_SCALE: tl.constexpr,
SHUFFLE: tl.constexpr,
BLOCK_TILE_SIZE: tl.constexpr,
HEAD_TILE_SIZE: tl.constexpr,
@@ -252,4 +257,10 @@ def _indexer_k_norm_rope_quant_store_kernel(
tl.store(val_ptr + tiled, x_uint8)
else:
tl.store(val_ptr + block, x_uint8)
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
if MX_SCALE:
tl.store(
scale_ptr.to(tl.pointer_type(tl.int32)),
pack_ue8m0_x4(tl.exp2(exponent)),
)
else:
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
@@ -10,7 +10,7 @@ import torch
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.forward_context import get_forward_context, override_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.v1.worker.gpu.cudagraph_utils import (
@@ -56,8 +56,9 @@ class DecoderReplayCudaGraphManager(CudaGraphManager):
config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype
assert isinstance(act, torch.dtype)
hc, h, f32 = config.hc_mult, config.hidden_size, torch.float32
mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
ids = torch.int64 if mega_moe else torch.int32
# DeepseekV4Model.forward casts input_ids to int64 for native MegaMoE only.
native = vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
ids = torch.int64 if native else torch.int32
# hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual
shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)]
dtypes = [act, torch.int64, ids, f32, f32, f32, act]
+8 -24
View File
@@ -19,10 +19,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -44,7 +40,7 @@ from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
@@ -409,12 +405,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -431,6 +421,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
else f".{self.linear_scale_name}"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -476,11 +473,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
@@ -492,17 +484,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
self.process_weights_after_loading()
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def _finalize_moe(self) -> None:
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
@@ -206,24 +206,36 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build, replay_start
)
num_tokens = metadata.num_decode_tokens
if not common_attn_metadata.causal and num_tokens > 0:
assert metadata.decode_swa_lens is not None
assert metadata.seq_lens is not None
assert metadata.token_to_req_indices is not None
topk_lens = self._decode_topk_lens[:num_tokens]
seq_lens = self._decode_seq_lens[:num_tokens]
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
torch.index_select(
metadata.seq_lens,
0,
metadata.token_to_req_indices[:num_tokens],
out=seq_lens,
)
metadata.flashinfer_decode_topk_lens = topk_lens
metadata.flashinfer_decode_seq_lens = seq_lens
if not common_attn_metadata.causal and metadata.num_decode_tokens > 0:
self._fill_noncausal_decode_lens(metadata)
return metadata
def update_draft_decode_metadata(
self, metadata: "DeepseekSparseSWAMetadata"
) -> None:
super().update_draft_decode_metadata(metadata)
if metadata.flashinfer_decode_topk_lens is not None:
self._fill_noncausal_decode_lens(metadata)
def _fill_noncausal_decode_lens(
self, metadata: "DeepseekSparseSWAMetadata"
) -> None:
num_tokens = metadata.num_decode_tokens
assert metadata.decode_swa_lens is not None
assert metadata.seq_lens is not None
assert metadata.token_to_req_indices is not None
topk_lens = self._decode_topk_lens[:num_tokens]
seq_lens = self._decode_seq_lens[:num_tokens]
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
torch.index_select(
metadata.seq_lens,
0,
metadata.token_to_req_indices[:num_tokens],
out=seq_lens,
)
metadata.flashinfer_decode_topk_lens = topk_lens
metadata.flashinfer_decode_seq_lens = seq_lens
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
@staticmethod
+9 -22
View File
@@ -15,7 +15,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_engram_dp_size,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
@@ -1251,14 +1250,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1275,6 +1266,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# Vision weights are loaded by the outer multimodal wrapper.
logger.warning_once("Skipping non-text weight: %s", name)
continue
if ".shared_experts." in name:
if is_pp_missing_parameter(name, self):
continue
experts = self.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if pad_shared_expert and ".shared_experts." in name:
loaded_weight = self._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
@@ -1337,14 +1337,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
@@ -1406,10 +1398,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
num_experts=self.config.n_routed_experts,
)
def finalize_mega_moe_weights(self) -> None:
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1680,7 +1668,6 @@ class DeepseekV41LLMForCausalLM(
return loaded_params
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+18 -18
View File
@@ -337,7 +337,7 @@ class DeepseekV41ModelState(DefaultModelState):
self._warm_up_replay_kernels(input_batch, slot_mappings)
window, cacheable_groups = self._replay
replay_start: torch.Tensor | None = None
if window:
if window or self.decoder_replay_layers is not None:
num_reqs = input_batch.num_reqs
# Decode rows sit above the hit, so only prefills carry a replay
# start; dummy batches (captures, profiling) carry none.
@@ -349,7 +349,7 @@ class DeepseekV41ModelState(DefaultModelState):
replay_start = self._replay_start_staging.copy_to_gpu(
replay_start_np, out=self._replay_start[:num_reqs]
)
if replay_start_np.any():
if window and replay_start_np.any():
# The replayed tokens rebuild window KV only: their slots in the
# prefix-cacheable groups are padded so the cached KV stays as is.
_pad_replayed_slots_kernel[(num_reqs,)](
@@ -397,21 +397,21 @@ class DeepseekV41ModelState(DefaultModelState):
dummy batches never replay or trim, so they would not launch them."""
assert self._replay is not None
window, cacheable_groups = self._replay
if not window:
return
_pad_replayed_slots_kernel.warmup(
slot_mappings,
slot_mappings.stride(0),
cacheable_groups,
input_batch.query_start_loc,
input_batch.positions,
self._replay_start,
window,
PAD_SLOT_ID,
NUM_GROUPS=cacheable_groups.numel(),
BLOCK=1024,
grid=(1,),
)
# Pad warmup is prefix-cache SWA. A zero window must not skip the gather warmup.
if window:
_pad_replayed_slots_kernel.warmup(
slot_mappings,
slot_mappings.stride(0),
cacheable_groups,
input_batch.query_start_loc,
input_batch.positions,
self._replay_start,
window,
PAD_SLOT_ID,
NUM_GROUPS=cacheable_groups.numel(),
BLOCK=1024,
grid=(1,),
)
if self.decoder_replay_layers is None:
return
assert self._kept_slot_mappings is not None
@@ -669,7 +669,7 @@ class DeepseekV41ModelState(DefaultModelState):
copy.create_metadata_builders(
self.vllm_config,
self.device,
group.metadata_builders[0].kernel_block_size,
group.kernel_block_size,
)
copies.append(copy)
own_groups.append(copies)
+116 -241
View File
@@ -228,13 +228,13 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
NC: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_t, i_c, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
i_i, i_j = i_c // NC, i_c % NC
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -255,58 +255,51 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
A += (bos * H + i_h) * BT
Aqk += (bos * H + i_h) * BT
p_b = tl.make_block_ptr(
beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)
)
b_b = tl.load(p_b, boundary_check=(0,))
o_i = i_t * BT + i_i * BC + tl.arange(0, BC)
o_j = i_t * BT + i_j * BC + tl.arange(0, BC)
m_i = o_i < T
m_j = o_j < T
p_b = beta + bos * H + i_h + o_i * H
b_b = tl.load(p_b, mask=m_i, other=0.0)
b_A = tl.zeros([BC, BC], dtype=tl.float32)
b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)
for i_k in range(tl.cdiv(K, BK)):
p_q = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
p_k = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
p_g = tl.make_block_ptr(
g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
b_kt = tl.make_block_ptr(
k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
)
p_gk = tl.make_block_ptr(
g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
)
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_qk = m_i[:, None] & m_k[None, :]
m_kj = m_k[:, None] & m_j[None, :]
p_q = q + o_i[:, None] * (H * K) + o_k[None, :]
p_k = k + o_i[:, None] * (H * K) + o_k[None, :]
p_g = g + o_i[:, None] * (H * K) + o_k[None, :]
b_kt = k + o_k[:, None] + o_j[None, :] * (H * K)
p_gk = g + o_k[:, None] + o_j[None, :] * (H * K)
# [BK,]
b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
# [BC, BK]
b_g = tl.load(p_g, boundary_check=(0, 1))
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
b_g = tl.load(p_g, mask=m_qk, other=0.0)
b_k = tl.load(p_k, mask=m_qk, other=0.0) * exp2(b_g - b_gn[None, :])
# [BK, BC]
b_gk = tl.load(p_gk, boundary_check=(0, 1))
b_kt = tl.load(b_kt, boundary_check=(0, 1))
b_gk = tl.load(p_gk, mask=m_kj, other=0.0)
b_kt = tl.load(b_kt, mask=m_kj, other=0.0)
# [BC, BC]
b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
b_A += tl.dot(b_k, b_ktg)
b_q = tl.load(p_q, boundary_check=(0, 1))
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
b_Aqk += tl.dot(b_qg, b_ktg)
b_A *= b_b[:, None]
p_A = tl.make_block_ptr(
A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
)
tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
p_Aqk = tl.make_block_ptr(
Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
)
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
o_jA = i_j * BC + tl.arange(0, BC)
m_A = m_i[:, None] & (o_jA[None, :] < BT)
p_A = A + o_i[:, None] * (H * BT) + o_jA[None, :]
tl.store(p_A, b_A.to(A.dtype.element_ty), mask=m_A)
p_Aqk = Aqk + o_i[:, None] * (H * BT) + o_jA[None, :]
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_A)
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
@@ -333,12 +326,12 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
BK: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -357,33 +350,14 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
m_A = (i_t * BT + i_i * BC + o_i) < T
o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
p_k = tl.make_block_ptr(
k + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
b_q = tl.load(p_q, boundary_check=(0, 1))
b_k = tl.load(p_k, boundary_check=(0, 1))
b_g = tl.load(p_g, boundary_check=(0, 1))
o_c = i_t * BT + i_i * BC + tl.arange(0, BC)
m_qk = m_A[:, None] & m_k[None, :]
p_q = q + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
p_k = k + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
p_g = g + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_k = tl.load(p_k, mask=m_qk, other=0.0)
b_g = tl.load(p_g, mask=m_qk, other=0.0)
p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]
@@ -536,12 +510,12 @@ def recompute_w_u_fwd_kernel(
IS_VARLEN: tl.constexpr,
DOT_PRECISION: tl.constexpr,
):
i_t, i_bh = tl.program_id(0), tl.program_id(1)
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -550,108 +524,57 @@ def recompute_w_u_fwd_kernel(
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
b_b = tl.load(p_b, boundary_check=(0,))
o_t = i_t * BT + tl.arange(0, BT)
m_t = o_t < T
p_b = beta + bos * H + i_h + o_t * H
b_b = tl.load(p_b, mask=m_t, other=0.0)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
b_A = tl.load(p_A, boundary_check=(0, 1))
o_A = tl.arange(0, BT)
m_A = m_t[:, None] & (o_A[None, :] < BT)
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
b_A = tl.load(p_A, mask=m_A, other=0.0)
for i_v in range(tl.cdiv(V, BV)):
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_u = tl.make_block_ptr(
u + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
b_v = tl.load(p_v, boundary_check=(0, 1))
o_v = i_v * BV + tl.arange(0, BV)
m_v = m_t[:, None] & (o_v[None, :] < V)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
b_v = tl.load(p_v, mask=m_v, other=0.0)
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
for i_k in range(tl.cdiv(K, BK)):
p_w = tl.make_block_ptr(
w + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_k = tl.make_block_ptr(
k + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_k = tl.load(p_k, boundary_check=(0, 1))
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_tk = m_t[:, None] & m_k[None, :]
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_k = tl.load(p_k, mask=m_tk, other=0.0)
b_kb = b_k * b_b[:, None]
p_gk = tl.make_block_ptr(
gk + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_gk = tl.load(p_gk, boundary_check=(0, 1))
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
b_kb *= exp2(b_gk)
if STORE_QG:
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_qg = tl.make_block_ptr(
qg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_q = tl.load(p_q, boundary_check=(0, 1))
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_q = tl.load(p_q, mask=m_tk, other=0.0)
b_qg = b_q * exp2(b_gk)
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
if STORE_KG:
last_idx = min(i_t * BT + BT, T) - 1
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
b_gn = tl.load(
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
)
b_kg = b_k * exp2(b_gn - b_gk)
p_kg = tl.make_block_ptr(
kg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
def recompute_w_u_fwd(
@@ -732,13 +655,13 @@ def chunk_gla_fwd_kernel_o(
BV: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_tg = i_t
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -754,70 +677,44 @@ def chunk_gla_fwd_kernel_o(
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
b_o = tl.zeros([BT, BV], dtype=tl.float32)
o_t = i_t * BT + tl.arange(0, BT)
o_v = i_v * BV + tl.arange(0, BV)
o_i = tl.arange(0, BT)
m_t = o_t < T
m_v = o_v < V
m_tv = m_t[:, None] & m_v[None, :]
m_A = m_t[:, None] & (o_i[None, :] < BT)
for i_k in range(tl.cdiv(K, BK)):
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_h = tl.make_block_ptr(
h + (i_tg * H + i_h) * K * V,
(V, K),
(K, 1),
(i_v * BV, i_k * BK),
(BV, BK),
(1, 0),
)
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_qk = m_t[:, None] & m_k[None, :]
m_h = m_v[:, None] & m_k[None, :]
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
# [BT, BK]
b_q = tl.load(p_q, boundary_check=(0, 1))
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_q = (b_q * scale).to(b_q.dtype)
# [BT, BK]
b_g = tl.load(p_g, boundary_check=(0, 1))
b_g = tl.load(p_g, mask=m_qk, other=0.0)
# [BT, BK]
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
# [BV, BK]
b_h = tl.load(p_h, boundary_check=(0, 1))
b_h = tl.load(p_h, mask=m_h, other=0.0)
# [BT, BV]
if i_k >= 0:
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_o = tl.make_block_ptr(
o + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
# [BT, BV]
b_v = tl.load(p_v, boundary_check=(0, 1))
b_v = tl.load(p_v, mask=m_tv, other=0.0)
# [BT, BT]
b_A = tl.load(p_A, boundary_check=(0, 1))
b_A = tl.load(p_A, mask=m_A, other=0.0)
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
b_o += tl.dot(b_A, b_v, allow_tf32=False)
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
def chunk_gla_fwd_o_gk(
@@ -896,12 +793,12 @@ def kda_gate_cumsum_fwd_kernel(
HAS_BIAS: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -911,26 +808,15 @@ def kda_gate_cumsum_fwd_kernel(
else:
bos = i_b * T
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * D,
(T, D),
(H * D, 1),
(i_t * BT, i_d * BD),
(BT, BD),
(1, 0),
)
p_y = tl.make_block_ptr(
y + (bos * H + i_h) * D,
(T, D),
(H * D, 1),
(i_t * BT, i_d * BD),
(BT, BD),
(1, 0),
)
o_t = i_t * BT + tl.arange(0, BT)
o_d = i_d * BD + tl.arange(0, BD)
m_t = o_t < T
m_g = m_t[:, None] & (o_d[None, :] < D)
p_g = g + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
p_y = y + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
b_g = tl.load(p_g, mask=m_g, other=0.0).to(tl.float32)
if HAS_BIAS:
o_d = i_d * BD + tl.arange(0, BD)
b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
b_g = b_g + b_bias[None, :]
@@ -951,12 +837,12 @@ def kda_gate_cumsum_fwd_kernel(
# Out-of-bounds rows (load returns 0, but softplus/bias can still make
# b_gate non-zero) participate in the dot product. They only contribute to
# out-of-bounds output rows, which are masked away by `boundary_check` on
# out-of-bounds output rows, which are masked away by `mask` on
# the store, so visible output matches unfused gate + chunk-local cumsum.
o_t = tl.arange(0, BT)
m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)
o_c = tl.arange(0, BT)
m_cumsum = tl.where(o_c[:, None] >= o_c[None, :], 1.0, 0.0)
b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_y, b_y.to(p_y.dtype.element_ty), mask=m_g)
def fused_kda_gate_chunk_cumsum(
@@ -1271,7 +1157,7 @@ def kda_gate_fwd_kernel(
BD: tl.constexpr,
HAS_BIAS: tl.constexpr,
):
i_t, i_h = tl.program_id(0), tl.program_id(1)
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
n_t = i_t * BT
b_a = tl.load(A + i_h).to(tl.float32)
@@ -1280,25 +1166,14 @@ def kda_gate_fwd_kernel(
stride_row = H * D
stride_col = 1
g_ptr = tl.make_block_ptr(
base=g + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
o_t = n_t + tl.arange(0, BT)
o_d = tl.arange(0, BD)
m_t = o_t < T
m_g = m_t[:, None] & (o_d[None, :] < D)
g_ptr = g + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
y_ptr = y + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
y_ptr = tl.make_block_ptr(
base=y + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
if HAS_BIAS:
n_d = tl.arange(0, BD)
@@ -1321,7 +1196,7 @@ def kda_gate_fwd_kernel(
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
b_y = b_a * sp
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
def fused_kda_gate(
+27 -10
View File
@@ -396,16 +396,33 @@ def sparse_attn_indexer_kpool(
chunk.token_start : chunk.token_end, :topk_tokens
]
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
if rocm_aiter_ops.is_indexer_top_k_supported(
is_prefill=True,
compress_ratio=index_kpool,
num_rows=num_rows,
):
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_launch_aiter_top_k_per_row_prefill,
)
_launch_aiter_top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
select_k,
)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
if index_kpool > 1:
pool_ids = pool_topk.to(torch.int64)
+13 -49
View File
@@ -33,9 +33,9 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import CircularBufferSpec, MLAAttentionSpec
logger = init_logger(__name__)
@@ -89,15 +89,9 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
The indexer shares one block with the co-located MLA (a single
``MLAAttentionSpec`` / block_table), so ``block_size`` is the model-wide
``cache_config.block_size``. DeepGEMM's paged-MQA kernel
(``csrc/apis/attention.hpp``) requires ``block_kv`` to be exactly 32 or
64, so the storage block is virtually split into pool pages of the
largest such size that tiles it (``storage_kernel_block_size``); this
needs ``block_size`` to be a multiple of ``index_kpool * 32`` (512 for
``index_kpool = 16``). A smaller block (e.g. the default 64) silently
collapses ``storage_block_size`` (64 // 16 = 4) and only fails later at
the opaque C++ assert; ``get_kv_cache_spec`` guards this up front
instead.
``cache_config.block_size``. DeepGEMM's paged-MQA kernel takes 32- or
64-state pages (only 64 on SM120), which ``Glm5NextIndexerBackend``
declares as its kernel block sizes.
"""
def __init__(
@@ -113,12 +107,6 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
)
assert index_kpool > 1, "Glm5NextIndexerCache expects index_kpool > 1"
# Keep chunked-prefill boundaries aligned to complete pools.
assert cache_config.block_size % index_kpool == 0, (
"Glm5NextIndexerCache: cache_config.block_size "
f"({cache_config.block_size}) must be a multiple of index_kpool "
f"({index_kpool}) so chunked-prefill boundaries stay pool-aligned."
)
self._index_kpool = index_kpool
def get_kv_cache_spec(self, vllm_config: VllmConfig):
@@ -128,33 +116,10 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
# ``tokens_per_state`` is the KV-spec representation of kpool
# compression in the current cache-layout API.
assert isinstance(spec, MLAAttentionSpec)
spec = replace(spec, tokens_per_state=self._index_kpool)
return replace(spec, tokens_per_state=self._index_kpool)
# DeepGEMM paged-MQA takes block_kv in {32, 64}; the storage block
# (= block_size // index_kpool) is virtually split into pool pages of
# the largest such size that tiles it, so it must be a multiple of 32.
storage_block_size = spec.block_size // self._index_kpool
assert (
spec.block_size % self._index_kpool == 0 and storage_block_size % 32 == 0
), (
"Glm5NextIndexerCache: kpool indexer requires cache block_size to "
f"be a multiple of index_kpool * 32 ({self._index_kpool * 32}) so "
"that DeepGEMM paged-MQA pool pages (32 or 64 entries) tile the "
f"storage block, got block_size={spec.block_size} -> "
f"storage_block_size={storage_block_size}."
)
max_page_size = max(PAGED_MQA_PAGE_SIZES)
min_page_size = min(PAGED_MQA_PAGE_SIZES)
if storage_block_size <= max_page_size:
page_size = storage_block_size
elif storage_block_size % max_page_size == 0:
page_size = max_page_size
else:
page_size = min_page_size
return replace(
spec,
storage_block_size=page_size * self._index_kpool,
)
def get_attn_backend(self):
return Glm5NextIndexerBackend
class Glm5NextTailCache(DeepseekV32IndexerCache):
@@ -164,9 +129,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
``ring`` slots per request, overwritten in place by ``pos % ring``
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
tail raw K+gate); the connector transfers it across PD; decode reads it to
compress the boundary pool correctly. ``KpoolTailSpec`` /
``KpoolTailManager`` provide the no-prune, 1-block/req allocation that lets
the in-progress pool survive across steps and across transfer.
compress the boundary pool correctly. ``CircularBufferSpec`` provides the
no-prune, 1-block/req allocation that lets the in-progress pool survive
across steps and across transfer.
Stores raw bf16 K (``head_dim``) as the "K" half of each block and the
bf16 gate score (``head_dim``) as the "V" half -- not the fp8-compressed
@@ -202,13 +167,12 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
f"({self.cache_config.block_size}) must be a multiple of the "
f"tail ring ({ring})"
)
return KpoolTailSpec(
return CircularBufferSpec(
block_size=ring,
num_kv_heads=2,
head_size=self.head_dim,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=ring,
# Tiny per-request ring; every DCP rank keeps the full tail so the
# rank owning a pool can compress it.
dcp_sharded=False,
@@ -303,7 +267,7 @@ class Indexer(nn.Module):
# Paged tail cache (in-progress pool's raw K + gate score). Written by
# prefill (seeds the boundary pool) and decode (per-step stash); read by
# the decode kernel to compress the boundary pool. Transferred across PD
# so the decode side sees the prefill tail. See KpoolTailSpec/Manager.
# so the decode side sees the prefill tail. See CircularBufferManager.
self.tail_cache = Glm5NextTailCache(
head_dim=self.head_dim,
dtype=torch.bfloat16,
+3 -1
View File
@@ -181,7 +181,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
return MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
def get_state_shape(
+3 -1
View File
@@ -1092,7 +1092,9 @@ class Glm5NextForCausalLM(
vllm_config: "VllmConfig",
) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.kda_state_dtype(
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
vllm_config.model_config.dtype,
vllm_config.cache_config.mamba_cache_dtype,
vllm_config.cache_config.mamba_ssm_cache_dtype,
)
@classmethod
@@ -15,6 +15,8 @@ from __future__ import annotations
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.ops.common import pack_ue8m0_x4
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
INDEX_HEAD_DIM = 128
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
HAS_WRITE_MASK: tl.constexpr,
RETURN_COMPRESSED: tl.constexpr,
WRITE_CACHE: tl.constexpr,
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
mask=do_write,
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if RETURN_COMPRESSED:
tl.store(
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
buf_fp8 = buf.view(torch.float8_e4m3fn)
buf_fp32 = buf.view(torch.float32)
mx_scale = write_cache and mqa_logits_use_mx_scales()
# bytes per page (last dim of kv_cache) viewed as uint8
buf_numel_per_page = buf.stride(0)
s_offset_nbytes_in_page = page_size * head_dim
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
POOL_SIZE=slot_k.shape[1],
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mx_scale,
MX_SCALE=mx_scale,
HAS_WRITE_MASK=has_write_mask,
RETURN_COMPRESSED=return_compressed,
WRITE_CACHE=write_cache,
@@ -396,11 +409,6 @@ def _kpool_tail_seed_kernel(
ahead belongs to a different tail block (or is past the batch / padding,
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
``tail[block, {0:K, 1:score}, pos % RING, :]``.
The tail cache aliases the indexer cache with the indexer's (padded) block
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
``KPOOL_HEAD`` (``tail.stride(0)`` / ``tail.stride(1)``), never as a dense
``[num_blocks, 2, KPOOL, HEAD_DIM]`` array.
"""
i = tl.program_id(0)
t = tl.load(tslot_ptr + i).to(tl.int64)
@@ -484,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""One program per request; iterates its NEXT_N verify tokens in order.
@@ -609,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
tl.store(buf_fp32_ptr + out_s_offset, scale)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale)
# Stash the current token AFTER any completion read so the completion
# uses prior stashes (and the current token's own key/score via
@@ -723,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
MX_SCALE=mqa_logits_use_mx_scales(),
BLOCK_D=triton.next_power_of_2(head_dim),
)
+116 -241
View File
@@ -230,13 +230,13 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
NC: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_t, i_c, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
i_i, i_j = i_c // NC, i_c % NC
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -257,58 +257,51 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
A += (bos * H + i_h) * BT
Aqk += (bos * H + i_h) * BT
p_b = tl.make_block_ptr(
beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)
)
b_b = tl.load(p_b, boundary_check=(0,))
o_i = i_t * BT + i_i * BC + tl.arange(0, BC)
o_j = i_t * BT + i_j * BC + tl.arange(0, BC)
m_i = o_i < T
m_j = o_j < T
p_b = beta + bos * H + i_h + o_i * H
b_b = tl.load(p_b, mask=m_i, other=0.0)
b_A = tl.zeros([BC, BC], dtype=tl.float32)
b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)
for i_k in range(tl.cdiv(K, BK)):
p_q = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
p_k = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
p_g = tl.make_block_ptr(
g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
)
b_kt = tl.make_block_ptr(
k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
)
p_gk = tl.make_block_ptr(
g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
)
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_qk = m_i[:, None] & m_k[None, :]
m_kj = m_k[:, None] & m_j[None, :]
p_q = q + o_i[:, None] * (H * K) + o_k[None, :]
p_k = k + o_i[:, None] * (H * K) + o_k[None, :]
p_g = g + o_i[:, None] * (H * K) + o_k[None, :]
b_kt = k + o_k[:, None] + o_j[None, :] * (H * K)
p_gk = g + o_k[:, None] + o_j[None, :] * (H * K)
# [BK,]
b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
# [BC, BK]
b_g = tl.load(p_g, boundary_check=(0, 1))
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
b_g = tl.load(p_g, mask=m_qk, other=0.0)
b_k = tl.load(p_k, mask=m_qk, other=0.0) * exp2(b_g - b_gn[None, :])
# [BK, BC]
b_gk = tl.load(p_gk, boundary_check=(0, 1))
b_kt = tl.load(b_kt, boundary_check=(0, 1))
b_gk = tl.load(p_gk, mask=m_kj, other=0.0)
b_kt = tl.load(b_kt, mask=m_kj, other=0.0)
# [BC, BC]
b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
b_A += tl.dot(b_k, b_ktg)
b_q = tl.load(p_q, boundary_check=(0, 1))
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
b_Aqk += tl.dot(b_qg, b_ktg)
b_A *= b_b[:, None]
p_A = tl.make_block_ptr(
A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
)
tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
p_Aqk = tl.make_block_ptr(
Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
)
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
o_jA = i_j * BC + tl.arange(0, BC)
m_A = m_i[:, None] & (o_jA[None, :] < BT)
p_A = A + o_i[:, None] * (H * BT) + o_jA[None, :]
tl.store(p_A, b_A.to(A.dtype.element_ty), mask=m_A)
p_Aqk = Aqk + o_i[:, None] * (H * BT) + o_jA[None, :]
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_A)
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
@@ -335,12 +328,12 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
BK: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -359,33 +352,14 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
m_A = (i_t * BT + i_i * BC + o_i) < T
o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
p_k = tl.make_block_ptr(
k + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT + i_i * BC, 0),
(BC, BK),
(1, 0),
)
b_q = tl.load(p_q, boundary_check=(0, 1))
b_k = tl.load(p_k, boundary_check=(0, 1))
b_g = tl.load(p_g, boundary_check=(0, 1))
o_c = i_t * BT + i_i * BC + tl.arange(0, BC)
m_qk = m_A[:, None] & m_k[None, :]
p_q = q + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
p_k = k + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
p_g = g + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_k = tl.load(p_k, mask=m_qk, other=0.0)
b_g = tl.load(p_g, mask=m_qk, other=0.0)
p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]
@@ -538,12 +512,12 @@ def recompute_w_u_fwd_kernel(
IS_VARLEN: tl.constexpr,
DOT_PRECISION: tl.constexpr,
):
i_t, i_bh = tl.program_id(0), tl.program_id(1)
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -552,108 +526,57 @@ def recompute_w_u_fwd_kernel(
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
b_b = tl.load(p_b, boundary_check=(0,))
o_t = i_t * BT + tl.arange(0, BT)
m_t = o_t < T
p_b = beta + bos * H + i_h + o_t * H
b_b = tl.load(p_b, mask=m_t, other=0.0)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
b_A = tl.load(p_A, boundary_check=(0, 1))
o_A = tl.arange(0, BT)
m_A = m_t[:, None] & (o_A[None, :] < BT)
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
b_A = tl.load(p_A, mask=m_A, other=0.0)
for i_v in range(tl.cdiv(V, BV)):
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_u = tl.make_block_ptr(
u + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
b_v = tl.load(p_v, boundary_check=(0, 1))
o_v = i_v * BV + tl.arange(0, BV)
m_v = m_t[:, None] & (o_v[None, :] < V)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
b_v = tl.load(p_v, mask=m_v, other=0.0)
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
for i_k in range(tl.cdiv(K, BK)):
p_w = tl.make_block_ptr(
w + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_k = tl.make_block_ptr(
k + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_k = tl.load(p_k, boundary_check=(0, 1))
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_tk = m_t[:, None] & m_k[None, :]
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_k = tl.load(p_k, mask=m_tk, other=0.0)
b_kb = b_k * b_b[:, None]
p_gk = tl.make_block_ptr(
gk + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_gk = tl.load(p_gk, boundary_check=(0, 1))
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
b_kb *= exp2(b_gk)
if STORE_QG:
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_qg = tl.make_block_ptr(
qg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_q = tl.load(p_q, boundary_check=(0, 1))
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_q = tl.load(p_q, mask=m_tk, other=0.0)
b_qg = b_q * exp2(b_gk)
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
if STORE_KG:
last_idx = min(i_t * BT + BT, T) - 1
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
b_gn = tl.load(
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
)
b_kg = b_k * exp2(b_gn - b_gk)
p_kg = tl.make_block_ptr(
kg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
def recompute_w_u_fwd(
@@ -734,13 +657,13 @@ def chunk_gla_fwd_kernel_o(
BV: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_tg = i_t
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -756,70 +679,44 @@ def chunk_gla_fwd_kernel_o(
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
b_o = tl.zeros([BT, BV], dtype=tl.float32)
o_t = i_t * BT + tl.arange(0, BT)
o_v = i_v * BV + tl.arange(0, BV)
o_i = tl.arange(0, BT)
m_t = o_t < T
m_v = o_v < V
m_tv = m_t[:, None] & m_v[None, :]
m_A = m_t[:, None] & (o_i[None, :] < BT)
for i_k in range(tl.cdiv(K, BK)):
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_h = tl.make_block_ptr(
h + (i_tg * H + i_h) * K * V,
(V, K),
(K, 1),
(i_v * BV, i_k * BK),
(BV, BK),
(1, 0),
)
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_qk = m_t[:, None] & m_k[None, :]
m_h = m_v[:, None] & m_k[None, :]
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
# [BT, BK]
b_q = tl.load(p_q, boundary_check=(0, 1))
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_q = (b_q * scale).to(b_q.dtype)
# [BT, BK]
b_g = tl.load(p_g, boundary_check=(0, 1))
b_g = tl.load(p_g, mask=m_qk, other=0.0)
# [BT, BK]
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
# [BV, BK]
b_h = tl.load(p_h, boundary_check=(0, 1))
b_h = tl.load(p_h, mask=m_h, other=0.0)
# [BT, BV]
if i_k >= 0:
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_o = tl.make_block_ptr(
o + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
# [BT, BV]
b_v = tl.load(p_v, boundary_check=(0, 1))
b_v = tl.load(p_v, mask=m_tv, other=0.0)
# [BT, BT]
b_A = tl.load(p_A, boundary_check=(0, 1))
b_A = tl.load(p_A, mask=m_A, other=0.0)
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
b_o += tl.dot(b_A, b_v, allow_tf32=False)
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
def chunk_gla_fwd_o_gk(
@@ -898,12 +795,12 @@ def kda_gate_cumsum_fwd_kernel(
HAS_BIAS: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -913,26 +810,15 @@ def kda_gate_cumsum_fwd_kernel(
else:
bos = i_b * T
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * D,
(T, D),
(H * D, 1),
(i_t * BT, i_d * BD),
(BT, BD),
(1, 0),
)
p_y = tl.make_block_ptr(
y + (bos * H + i_h) * D,
(T, D),
(H * D, 1),
(i_t * BT, i_d * BD),
(BT, BD),
(1, 0),
)
o_t = i_t * BT + tl.arange(0, BT)
o_d = i_d * BD + tl.arange(0, BD)
m_t = o_t < T
m_g = m_t[:, None] & (o_d[None, :] < D)
p_g = g + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
p_y = y + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
b_g = tl.load(p_g, mask=m_g, other=0.0).to(tl.float32)
if HAS_BIAS:
o_d = i_d * BD + tl.arange(0, BD)
b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
b_g = b_g + b_bias[None, :]
@@ -953,12 +839,12 @@ def kda_gate_cumsum_fwd_kernel(
# Out-of-bounds rows (load returns 0, but softplus/bias can still make
# b_gate non-zero) participate in the dot product. They only contribute to
# out-of-bounds output rows, which are masked away by `boundary_check` on
# out-of-bounds output rows, which are masked away by `mask` on
# the store, so visible output matches unfused gate + chunk-local cumsum.
o_t = tl.arange(0, BT)
m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)
o_c = tl.arange(0, BT)
m_cumsum = tl.where(o_c[:, None] >= o_c[None, :], 1.0, 0.0)
b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_y, b_y.to(p_y.dtype.element_ty), mask=m_g)
def fused_kda_gate_chunk_cumsum(
@@ -1273,7 +1159,7 @@ def kda_gate_fwd_kernel(
BD: tl.constexpr,
HAS_BIAS: tl.constexpr,
):
i_t, i_h = tl.program_id(0), tl.program_id(1)
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
n_t = i_t * BT
b_a = tl.load(A + i_h).to(tl.float32)
@@ -1282,25 +1168,14 @@ def kda_gate_fwd_kernel(
stride_row = H * D
stride_col = 1
g_ptr = tl.make_block_ptr(
base=g + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
o_t = n_t + tl.arange(0, BT)
o_d = tl.arange(0, BD)
m_t = o_t < T
m_g = m_t[:, None] & (o_d[None, :] < D)
g_ptr = g + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
y_ptr = y + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
y_ptr = tl.make_block_ptr(
base=y + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
if HAS_BIAS:
n_d = tl.arange(0, BD)
@@ -1323,7 +1198,7 @@ def kda_gate_fwd_kernel(
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
b_y = b_a * sp
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
def fused_kda_gate(
+80 -43
View File
@@ -12,7 +12,12 @@ from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
from vllm.model_executor.layers.sparse_attn_indexer import (
PrefillRowShard,
_merge_dcp_topk_global,
_reserve_row_shard_exchange_transient,
as_fp32_logits,
)
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
@@ -25,7 +30,13 @@ from vllm.models.glm5next.common.sparse_indexer import (
)
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import has_deep_gemm
from vllm.utils.deep_gemm import (
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
LayerNameType,
@@ -169,6 +180,10 @@ def sparse_attn_indexer_kpool(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
# The dummy run returns before exchange_topk executes, so reserve its
# worst-case transient for the memory profile.
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
return topk_indices_buffer
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
@@ -245,7 +260,7 @@ def sparse_attn_indexer_kpool(
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
@@ -282,6 +297,12 @@ def sparse_attn_indexer_kpool(
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
_fill_causal_indices(_buf, _pos)
# Short prefills score nothing locally; suppress the shard there.
row_shard = None
if not short_prefill:
row_shard = PrefillRowShard.from_metadata(
prefill_metadata, num_decode_tokens
)
# Get the full shared workspace buffers once (will allocate on first use).
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
@@ -310,12 +331,16 @@ def sparse_attn_indexer_kpool(
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
row_start, row_end = chunk.token_start, chunk.token_end
cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke
if row_shard is not None:
narrowed = row_shard.narrow(chunk)
if narrowed is None:
continue
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
q_slice = q_quant[row_start:row_end]
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
@@ -325,20 +350,29 @@ def sparse_attn_indexer_kpool(
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
k_scale_cast = k_scale.view(
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
).squeeze(-1)
num_rows = q_slice.shape[0]
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
q_slice_cast, q_scale_slice, weights[row_start:row_end]
)
# Compressed: row i starts at K position cu_seqlen_ks[i].
row_ends = cu_seqlen_ke - cu_seqlen_ks
logits = as_fp32_logits(
fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
chunk_weights,
cu_seqlen_ks,
cu_seqlen_ke,
chunk.max_seqlen_k,
),
row_ends,
)
# kpool: logits are pool-granular (compress_ratio == index_kpool),
@@ -351,17 +385,15 @@ def sparse_attn_indexer_kpool(
)
topk_dst = pool_topk
else:
topk_dst = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens]
if logits.shape[1] == 0:
topk_dst.fill_(-1)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
torch.zeros_like(row_ends),
row_ends,
topk_dst,
num_rows,
logits.stride(0),
@@ -376,7 +408,7 @@ def sparse_attn_indexer_kpool(
dcp_rank,
dcp_world_size,
pool_cp_interleave,
row_starts=chunk.cu_seqlen_ks,
row_starts=None,
)
if index_kpool > 1:
@@ -385,10 +417,7 @@ def sparse_attn_indexer_kpool(
# Fused expand-pools + append-tail into one Triton kernel
# (replaces ~25 elementwise ops). seq_len is token-granular
# (pos+1); the kernel derives pool_len internally.
q_seq = (
positions[chunk.token_start : chunk.token_end].to(torch.int32)
+ 1
)
q_seq = positions[row_start:row_end].to(torch.int32) + 1
expanded = kpool_ops.expand_pools_and_append_tail(
pool_ids, q_seq, index_kpool
)
@@ -397,9 +426,13 @@ def sparse_attn_indexer_kpool(
expanded = kpool_ops.expand_pools_to_tokens(
pool_ids, valid, topk_tokens, index_kpool
)
topk_indices_buffer[
chunk.token_start : chunk.token_end, : expanded.shape[-1]
] = expanded
topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded
if row_shard is not None:
# The k-pool expansion appends the request's incomplete tail
# (index_kpool - 1 entries) after the logical top-k history; those
# columns are part of the attention index and are exchanged too.
row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
@@ -577,18 +610,22 @@ def sparse_attn_indexer_kpool(
if use_fp4_cache
else padded_q_quant_decode_tokens
)
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_pool_len,
clean_logits=False,
indices=decode_metadata.indices,
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
padded_q_quant_cast, padded_q_scale, padded_weights[:num_padded_tokens]
)
# SM100 decode is varlen: one row and context length per token.
logits = as_fp32_logits(
fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
decode_weights,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_pool_len,
indices=decode_metadata.indices,
),
seq_lens.view(-1),
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
+26 -46
View File
@@ -36,8 +36,6 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
@@ -435,12 +433,6 @@ class HYV4Model(nn.Module):
split_expert_params_mapping = self.get_expert_mapping()
loaded_params: set[str] = set()
# Sink weights are sharded like the q/k/v linears.
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
@@ -572,48 +564,36 @@ class HYV4Model(nn.Module):
if loaded_expert_param_names:
loaded_params.update(loaded_expert_param_names)
continue
if "learnable_sink_param" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
else:
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace(
"gate.e_score_correction_bias", "expert_bias"
)
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace("gate.e_score_correction_bias", "expert_bias")
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_pp_missing_parameter(name, self):
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
if _should_skip_missing_param(name):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if name not in params_dict:
if _should_skip_missing_param(name):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
return loaded_params
+6 -7
View File
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionBackend, AttentionType
@@ -493,13 +494,11 @@ class HYV4MLAAttention(nn.Module):
if self.learnable_sink:
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
enable_sink = sink_backend is not None
self.learnable_sink_param = nn.Parameter(
torch.empty(
self.num_local_heads,
# The kernels require fp32 sinks; the disabled path keeps
# the checkpoint dtype since the value is never consumed.
dtype=torch.float32 if enable_sink else torch.bfloat16,
)
# The kernels require fp32 sinks; the disabled path keeps the
# checkpoint dtype since the value is never consumed.
self.learnable_sink_param = make_attention_sink(
self.num_local_heads,
torch.float32 if enable_sink else torch.bfloat16,
)
if enable_sink:
sinks = self.learnable_sink_param
+26 -46
View File
@@ -35,8 +35,6 @@ from transformers import PreTrainedConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
@@ -433,12 +431,6 @@ class HYV4Model(nn.Module):
split_expert_params_mapping = self.get_expert_mapping()
loaded_params: set[str] = set()
# Sink weights are sharded like the q/k/v linears.
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
@@ -570,48 +562,36 @@ class HYV4Model(nn.Module):
if loaded_expert_param_names:
loaded_params.update(loaded_expert_param_names)
continue
if "learnable_sink_param" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
else:
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace(
"gate.e_score_correction_bias", "expert_bias"
)
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace("gate.e_score_correction_bias", "expert_bias")
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_pp_missing_parameter(name, self):
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
if _should_skip_missing_param(name):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if name not in params_dict:
if _should_skip_missing_param(name):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
return loaded_params
-18
View File
@@ -11,10 +11,6 @@ from torch import nn
from transformers import PreTrainedConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -744,11 +740,6 @@ class HYV4MTP(nn.Module):
}
num_experts = getattr(self.config, "n_routed_experts", 0)
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
# Routed-expert weights come in two checkpoint layouts:
# split: mlp.experts.<id>.gate_proj.weight
@@ -883,15 +874,6 @@ class HYV4MTP(nn.Module):
if is_loaded:
continue
if "learnable_sink_param" in name:
if name in params_dict:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
@@ -165,15 +165,23 @@ def _inkling_rel_attention_kernel(
q_head_start = pid_kh * gqa_group_size
bt_row = block_table_ptr + pid_b * stride_bt_b
q_block_ptr = tl.make_block_ptr(
base=q_ptr + q_start * stride_q_t + q_head_start * stride_q_h,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_q_t, stride_q_h, stride_q_d),
offsets=(q_block, 0, 0),
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
order=(2, 1, 0),
o0 = q_block + tl.arange(0, BLOCK_Q)
o1 = tl.arange(0, BLOCK_H)
o2 = tl.arange(0, BLOCK_D)
m_q = (
(o0 < q_len)[:, None, None]
& (o1 < gqa_group_size)[None, :, None]
& (o2 < head_dim)[None, None, :]
)
q = tl.load(q_block_ptr, boundary_check=(0, 1, 2), padding_option="zero")
q_block_ptr = (
q_ptr
+ q_start * stride_q_t
+ q_head_start * stride_q_h
+ o0[:, None, None] * stride_q_t
+ o1[None, :, None] * stride_q_h
+ o2[None, None, :] * stride_q_d
)
q = tl.load(q_block_ptr, mask=m_q, other=0.0)
q = tl.reshape(q, (BLOCK_QH, BLOCK_D))
q_rows = q_block + tl.arange(0, BLOCK_Q)
@@ -265,18 +273,18 @@ def _inkling_rel_attention_kernel(
acc /= l_i[:, None]
acc = tl.reshape(acc, (BLOCK_Q, BLOCK_H, BLOCK_D))
out_block_ptr = tl.make_block_ptr(
base=out_ptr + q_start * stride_o_t + q_head_start * stride_o_h,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_o_t, stride_o_h, stride_o_d),
offsets=(q_block, 0, 0),
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
order=(2, 1, 0),
out_block_ptr = (
out_ptr
+ q_start * stride_o_t
+ q_head_start * stride_o_h
+ o0[:, None, None] * stride_o_t
+ o1[None, :, None] * stride_o_h
+ o2[None, None, :] * stride_o_d
)
tl.store(
out_block_ptr,
acc.to(out_ptr.dtype.element_ty),
boundary_check=(0, 1, 2),
mask=m_q,
)
+19 -22
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
@@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import (
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
is_fused_kda_decode_supported,
make_decode_conv1d_weight_loader,
make_decode_norm_weight_loader,
)
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
@@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.register_buffer(
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
)
if decode_conv1d_weight is None:
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
else:
conv1d_weight_loader = make_decode_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
decode_conv1d_weight,
)
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader})
self.A_log = nn.Parameter(
@@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
dtype=torch.float32,
device=self.o_norm.weight.device,
)
if hasattr(self.o_norm.weight, "weight_loader"):
delattr(self.o_norm.weight, "weight_loader")
set_weight_attrs(
self.o_norm.weight,
{"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)},
)
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
self.o_proj = RowParallelLinear(
self.projection_size,
@@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@torch.no_grad()
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
if is_weights_pre_processed():
return
if self.decode_conv1d_weight is not None:
self.decode_conv1d_weight.copy_(
self.conv1d.weight.view(
3, self.local_projection_size, self.conv_size
).transpose(1, 2)
)
if self.decode_norm_weight is not None:
self.decode_norm_weight.copy_(self.o_norm.weight)
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+3 -2
View File
@@ -50,8 +50,9 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
if attn.hisparse_cache is not None or attn.q_pad_num_heads is not None:
return False
# Only the W_UK variants reproduced in _fused_decode are supported.
return attn.is_aiter_triton_fp4_bmm_enabled or (
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
return (
attn.is_aiter_triton_fp4_bmm_enabled
or not attn.is_aiter_triton_fp8_bmm_enabled
)
def get_input_quant_key(self) -> QuantKey | None:
+1 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
The kernel in ``csrc/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
path with a single launch that keeps the per-chunk state in registers, so the
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
+3 -53
View File
@@ -2,22 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm entry points for the fused Kimi-K3 KDA decode kernel.
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_decode_kernel_rocm.cu``
The kernel in ``csrc/kimi_k3/fused_kda_decode_kernel_rocm.cu``
replaces, for a pure non-speculative decode batch, the three Triton launches
and two copies the AMD KDA layer otherwise runs per layer: the packed causal
conv1d update, the recurrent delta-rule step, and the gated output RMSNorm.
The kernel wants a width-major conv weight and an fp32 norm weight, so both are
staged once at load time by the weight loaders below.
The kernel wants a width-major conv weight and an fp32 norm weight, so the
layer stages both once in ``process_weights_after_loading``.
"""
from collections.abc import Callable
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
logger = init_logger(__name__)
@@ -51,50 +48,3 @@ def is_fused_kda_decode_supported(
# gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP /
# bf16 primitives the kernel relies on.
return on_gfx950() or on_gfx942()
def make_decode_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
decode_conv1d_weight: torch.Tensor | None,
) -> Callable[..., None]:
"""Load the packed conv1d weight, mirroring a width-major fp32 copy.
The fused kernel indexes the weight as ``[qkv, width, channel]`` so the
channel dimension is the contiguous one; the Triton prefill and fallback
decode kernels keep the ``[channel, width]`` layout.
"""
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
if decode_conv1d_weight is not None and not param.is_meta:
decode_conv1d_weight[loaded_shard_id].copy_(
loaded_shard.squeeze(1).transpose(0, 1)
)
return weight_loader
def make_decode_norm_weight_loader(
decode_norm_weight: torch.Tensor,
) -> Callable[..., None]:
"""Load the gated-norm weight, mirroring an fp32 copy for the kernel."""
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
default_weight_loader(param, loaded_weight)
if not param.is_meta:
decode_norm_weight.copy_(param.data)
return weight_loader
+1
View File
@@ -456,6 +456,7 @@ def _flashinfer_kda_prefill(
beta_is_logit=True,
seq_order=seq_order,
prefill_workspace=prefill_workspace,
backend="cute-dsl",
)
return output, initial_state
+4 -69
View File
@@ -95,10 +95,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4MegaMoEExperts,
DeepseekV4MLP,
)
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.nvidia.latent_moe_runner import (
@@ -351,10 +348,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
self.activation = activation
self.activation_beta = activation_beta
self.activation_linear_beta = activation_linear_beta
self.register_buffer("_mega_l1_packed", None, persistent=False)
self.register_buffer("_mega_l1_scale", None, persistent=False)
self.register_buffer("_mega_l2_packed", None, persistent=False)
self.register_buffer("_mega_l2_scale", None, persistent=False)
def synchronize_first_launch(self) -> None:
ep_group = get_ep_group()
@@ -366,51 +359,11 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
torch.distributed.barrier(group=ep_group.cpu_group)
self._synchronized_ep_groups.add(key)
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
if self._transformed_l1_weights is not None:
return
# Weight cache IPC engine: the daemon exported the transformed
# buffers; reuse them zero-copy and drop the raw packed params.
if self._mega_l1_packed is not None:
self._transformed_l1_weights = (self._mega_l1_packed, self._mega_l1_scale)
self._transformed_l2_weights = (self._mega_l2_packed, self._mega_l2_scale)
self._drop_raw_mega_weights()
return
self._check_runtime_supported()
backend = self._ensure_backend()
self._transformed_l1_weights, self._transformed_l2_weights = (
backend.transform_weights(
w13_weight=self.w13_weight.data,
w13_weight_scale=self.w13_weight_scale.data,
w2_weight=self.w2_weight.data,
w2_weight_scale=self.w2_weight_scale.data,
num_local_experts=self.num_local_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
activation=self.activation,
)
)
l1_packed, l1_scale = self._transformed_l1_weights
l2_packed, l2_scale = self._transformed_l2_weights
self.register_buffer("_mega_l1_packed", l1_packed, persistent=False)
self.register_buffer("_mega_l1_scale", l1_scale, persistent=False)
self.register_buffer("_mega_l2_packed", l2_packed, persistent=False)
self.register_buffer("_mega_l2_scale", l2_scale, persistent=False)
self._drop_raw_mega_weights()
def _drop_raw_mega_weights(self) -> None:
self.w13_weight = None
self.w13_weight_scale = None
self.w2_weight = None
self.w2_weight_scale = None
def get_symm_buffer(self):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
backend = self._ensure_backend()
backend = self.backend
group = get_ep_group().device_group
device = torch.accelerator.current_device_index()
key = (
@@ -485,7 +438,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
else None,
)
backend = self._ensure_backend()
backend = self.backend
symm_buffer = self.get_symm_buffer()
prepare_megamoe_inputs(
hidden_states,
@@ -498,9 +451,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
is_padding=is_padding,
hidden_quant=backend.hidden_quant,
)
self.finalize_weights()
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
@@ -566,6 +516,7 @@ class KimiMoE(nn.Module):
)
self.latent_moe_use_norm = config.latent_moe_use_norm
self.tp_size = get_tensor_model_parallel_world_size()
self.use_sequence_parallel = use_sequence_parallel
self.routed_scaling_factor = config.routed_scaling_factor
self.moe_renormalize = moe_renormalize
self.use_grouped_topk = config.use_grouped_topk
@@ -1572,11 +1523,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
loaded_params.add(name)
return loaded_params
def finalize_mega_moe_weights(self) -> None:
for module in self.modules():
if isinstance(module, KimiMoE) and module.use_mega_moe:
module.experts.finalize_weights()
class KimiLinearForCausalLM(
nn.Module,
@@ -1699,14 +1645,6 @@ class KimiLinearForCausalLM(
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def process_weights_after_loading(self) -> None:
# A parent AutoWeightsLoader may invoke load_weights repeatedly for
# non-contiguous streamed prefixes. Finalize only after the full stream.
self.model.finalize_mega_moe_weights()
# The fused MultiHeadLatentAttention's process_weights_after_loading
# (W_UK_T / W_UV absorption) is driven by the loader's generic post-load
# hook for any AttentionLayerBase, so no manual trigger is needed here.
def get_spec_layer_idx_from_weight_name(
config: KimiLinearConfig, weight_name: str
@@ -2153,6 +2091,3 @@ class KimiK3ForConditionalGeneration(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
self.language_model.process_weights_after_loading()
-5
View File
@@ -421,11 +421,6 @@ class KimiK3MTP(nn.Module):
f"MTP layer weights, or disable speculative decoding."
)
if use_mega_moe:
for module in self.modules():
if isinstance(module, KimiMoE) and module.use_mega_moe:
module.experts.finalize_weights()
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
@@ -3,7 +3,7 @@
"""Fused MLA prefill and decode epilogues for Kimi-K3.
Thin wrappers over the CUDA ops in
``csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu``, which
``csrc/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu``, which
mirror ``fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_{bf16,fp8}_insert``.
- ``fused_mla_key_concat_kv_cache_insert`` (bf16): optionally apply RoPE,
+75 -185
View File
@@ -69,12 +69,12 @@ def recompute_w_u_fwd_kernel(
IS_VARLEN: tl.constexpr,
DOT_PRECISION: tl.constexpr,
):
i_t, i_bh = tl.program_id(0), tl.program_id(1)
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -83,108 +83,57 @@ def recompute_w_u_fwd_kernel(
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32)
o_t = i_t * BT + tl.arange(0, BT)
m_t = o_t < T
p_b = beta + bos * H + i_h + o_t * H
b_b = tl.load(p_b, mask=m_t, other=0.0).to(tl.float32)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
b_A = tl.load(p_A, boundary_check=(0, 1))
o_A = tl.arange(0, BT)
m_A = m_t[:, None] & (o_A[None, :] < BT)
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
b_A = tl.load(p_A, mask=m_A, other=0.0)
for i_v in range(tl.cdiv(V, BV)):
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_u = tl.make_block_ptr(
u + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
b_v = tl.load(p_v, boundary_check=(0, 1))
o_v = i_v * BV + tl.arange(0, BV)
m_v = m_t[:, None] & (o_v[None, :] < V)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
b_v = tl.load(p_v, mask=m_v, other=0.0)
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
for i_k in range(tl.cdiv(K, BK)):
p_w = tl.make_block_ptr(
w + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_k = tl.make_block_ptr(
k + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_k = tl.load(p_k, boundary_check=(0, 1))
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_tk = m_t[:, None] & m_k[None, :]
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_k = tl.load(p_k, mask=m_tk, other=0.0)
b_kb = b_k * b_b[:, None]
p_gk = tl.make_block_ptr(
gk + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_gk = tl.load(p_gk, boundary_check=(0, 1))
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
b_kb *= exp2(b_gk)
if STORE_QG:
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_qg = tl.make_block_ptr(
qg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
b_q = tl.load(p_q, boundary_check=(0, 1))
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
b_q = tl.load(p_q, mask=m_tk, other=0.0)
b_qg = b_q * exp2(b_gk)
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
if STORE_KG:
last_idx = min(i_t * BT + BT, T) - 1
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
b_gn = tl.load(
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
)
b_kg = b_k * exp2(b_gn - b_gk)
p_kg = tl.make_block_ptr(
kg + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
def recompute_w_u_fwd(
@@ -265,13 +214,13 @@ def chunk_gla_fwd_kernel_o(
BV: tl.constexpr,
IS_VARLEN: tl.constexpr,
):
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_tg = i_t
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -287,70 +236,44 @@ def chunk_gla_fwd_kernel_o(
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
b_o = tl.zeros([BT, BV], dtype=tl.float32)
o_t = i_t * BT + tl.arange(0, BT)
o_v = i_v * BV + tl.arange(0, BV)
o_i = tl.arange(0, BT)
m_t = o_t < T
m_v = o_v < V
m_tv = m_t[:, None] & m_v[None, :]
m_A = m_t[:, None] & (o_i[None, :] < BT)
for i_k in range(tl.cdiv(K, BK)):
p_q = tl.make_block_ptr(
q + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_g = tl.make_block_ptr(
g + (bos * H + i_h) * K,
(T, K),
(H * K, 1),
(i_t * BT, i_k * BK),
(BT, BK),
(1, 0),
)
p_h = tl.make_block_ptr(
h + (i_tg * H + i_h) * K * V,
(V, K),
(K, 1),
(i_v * BV, i_k * BK),
(BV, BK),
(1, 0),
)
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
m_qk = m_t[:, None] & m_k[None, :]
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
m_h = m_v[:, None] & m_k[None, :]
# [BT, BK]
b_q = tl.load(p_q, boundary_check=(0, 1))
b_q = tl.load(p_q, mask=m_qk, other=0.0)
b_q = (b_q * scale).to(b_q.dtype)
# [BT, BK]
b_g = tl.load(p_g, boundary_check=(0, 1))
b_g = tl.load(p_g, mask=m_qk, other=0.0)
# [BT, BK]
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
# [BV, BK]
b_h = tl.load(p_h, boundary_check=(0, 1))
b_h = tl.load(p_h, mask=m_h, other=0.0)
# [BT, BV]
if i_k >= 0:
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
p_v = tl.make_block_ptr(
v + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_o = tl.make_block_ptr(
o + (bos * H + i_h) * V,
(T, V),
(H * V, 1),
(i_t * BT, i_v * BV),
(BT, BV),
(1, 0),
)
p_A = tl.make_block_ptr(
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
)
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
# [BT, BV]
b_v = tl.load(p_v, boundary_check=(0, 1))
b_v = tl.load(p_v, mask=m_tv, other=0.0)
# [BT, BT]
b_A = tl.load(p_A, boundary_check=(0, 1))
b_A = tl.load(p_A, mask=m_A, other=0.0)
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
b_o += tl.dot(b_A, b_v, allow_tf32=False)
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
def chunk_gla_fwd_o_gk(
@@ -434,12 +357,12 @@ def kda_gate_chunk_cumsum_vector_kernel(
IS_VARLEN: tl.constexpr,
USE_LOWER_BOUND: tl.constexpr,
):
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
i_b, i_h = i_bh // H, i_bh % H
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -472,34 +395,15 @@ def kda_gate_chunk_cumsum_vector_kernel(
i_s -= 1
p_s = tl.make_block_ptr(
s + (bos * H + i_h) * S,
(T, S),
(H * S, 1),
(i_t * BT, i_s * BS),
(BT, BS),
(1, 0),
)
p_o = tl.make_block_ptr(
o + (bos * H + i_h) * S,
(T, S),
(H * S, 1),
(i_t * BT, i_s * BS),
(BT, BS),
(1, 0),
)
o_t = i_t * BT + tl.arange(0, BT)
o_s = i_s * BS + tl.arange(0, BS)
m_s = (o_t[:, None] < T) & (o_s[None, :] < S)
p_s = s + (bos * H + i_h) * S + o_t[:, None] * (H * S) + o_s[None, :]
p_o = o + (bos * H + i_h) * S + o_t[:, None] * (H * S) + o_s[None, :]
b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32)
b_s = tl.load(p_s, mask=m_s, other=0.0).to(tl.float32)
if HAS_BIAS:
p_bias = tl.make_block_ptr(
g_bias + i_h * S,
(S,),
(1,),
(i_s * BS,),
(BS,),
(0,),
)
b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32)
b_bias = tl.load(g_bias + i_h * S + o_s, mask=o_s < S, other=0.0).to(tl.float32)
b_s += b_bias[None, :]
b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32))
@@ -517,7 +421,7 @@ def kda_gate_chunk_cumsum_vector_kernel(
# Boundary loads return zero, but bias and gate activation can make padded
# rows nonzero. Padding trails valid rows, so it only affects masked stores.
b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_s)
def fused_kda_gate_chunk_cumsum(
@@ -843,33 +747,19 @@ def kda_gate_fwd_kernel(
HAS_BIAS: tl.constexpr,
USE_LOWER_BOUND: tl.constexpr,
):
i_t, i_h = tl.program_id(0), tl.program_id(1)
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
n_t = i_t * BT
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
stride_row = H * D
stride_col = 1
o_t = n_t + tl.arange(0, BT)
o_d = tl.arange(0, BD)
m_t = o_t < T
m_g = m_t[:, None] & (o_d[None, :] < D)
g_ptr = g + i_h * D + o_t[:, None] * (H * D) + o_d[None, :]
y_ptr = y + i_h * D + o_t[:, None] * (H * D) + o_d[None, :]
g_ptr = tl.make_block_ptr(
base=g + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
y_ptr = tl.make_block_ptr(
base=y + i_h * D,
shape=(T, D),
strides=(stride_row, stride_col),
offsets=(n_t, 0),
block_shape=(BT, BD),
order=(1, 0),
)
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
if HAS_BIAS:
n_d = tl.arange(0, BD)
@@ -887,7 +777,7 @@ def kda_gate_fwd_kernel(
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
b_y = -b_a * sp
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
def fused_kda_gate(
+103 -167
View File
@@ -70,14 +70,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
5. Computes merged Akk_inv
6. Writes Akk_inv to Akk
"""
i_t, i_bh = tl.program_id(0), tl.program_id(1)
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
i_b, i_hv = i_bh // HV, i_bh % HV
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -106,6 +106,15 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
m_tc1 = (i_tc1 + o_i) < T
m_tc2 = (i_tc2 + o_i) < T
m_tc3 = (i_tc3 + o_i) < T
o_c0 = i_tc0 + o_i
o_c1 = i_tc1 + o_i
o_c2 = i_tc2 + o_i
o_c3 = i_tc3 + o_i
m_tc0 = o_c0 < T
m_A0 = m_tc0[:, None] & (o_i[None, :] < BT)
m_A1 = m_tc1[:, None] & (o_i[None, :] < BT)
m_A2 = m_tc2[:, None] & (o_i[None, :] < BT)
m_A3 = m_tc3[:, None] & (o_i[None, :] < BT)
b_Aqk10 = tl.zeros([BC, BC], dtype=tl.float32)
b_Akk10 = tl.zeros([BC, BC], dtype=tl.float32)
@@ -129,29 +138,21 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
p_k0 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
p_g0 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
m_ck0 = m_tc0[:, None] & m_k[None, :]
p_k0 = k + o_c0[:, None] * (H * K) + o_k[None, :]
p_g0 = g + o_c0[:, None] * (HV * K) + o_k[None, :]
b_k0 = tl.load(p_k0, mask=m_ck0, other=0.0).to(tl.float32)
b_g0 = tl.load(p_g0, mask=m_ck0, other=0.0).to(tl.float32)
if i_tc1 < T:
p_q1 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_k1 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_g1 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
m_ck1 = m_tc1[:, None] & m_k[None, :]
p_q1 = q + o_c1[:, None] * (H * K) + o_k[None, :]
p_k1 = k + o_c1[:, None] * (H * K) + o_k[None, :]
p_g1 = g + o_c1[:, None] * (HV * K) + o_k[None, :]
# [BC, BK]
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
b_q1 = tl.load(p_q1, mask=m_ck1, other=0.0).to(tl.float32)
b_k1 = tl.load(p_k1, mask=m_ck1, other=0.0).to(tl.float32)
b_g1 = tl.load(p_g1, mask=m_ck1, other=0.0).to(tl.float32)
# [BK]
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
# [BC, BK]
@@ -163,19 +164,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
if i_tc2 < T:
p_q2 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_k2 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_g2 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
m_ck2 = m_tc2[:, None] & m_k[None, :]
p_q2 = q + o_c2[:, None] * (H * K) + o_k[None, :]
p_k2 = k + o_c2[:, None] * (H * K) + o_k[None, :]
p_g2 = g + o_c2[:, None] * (HV * K) + o_k[None, :]
# [BC, BK]
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
b_q2 = tl.load(p_q2, mask=m_ck2, other=0.0).to(tl.float32)
b_k2 = tl.load(p_k2, mask=m_ck2, other=0.0).to(tl.float32)
b_g2 = tl.load(p_g2, mask=m_ck2, other=0.0).to(tl.float32)
# [BK]
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
@@ -195,19 +191,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk21 += tl.dot(b_kg2, b_kgt)
if i_tc3 < T:
p_q3 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_k3 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_g3 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
m_ck3 = m_tc3[:, None] & m_k[None, :]
p_q3 = q + o_c3[:, None] * (H * K) + o_k[None, :]
p_k3 = k + o_c3[:, None] * (H * K) + o_k[None, :]
p_g3 = g + o_c3[:, None] * (HV * K) + o_k[None, :]
# [BC, BK]
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
b_q3 = tl.load(p_q3, mask=m_ck3, other=0.0).to(tl.float32)
b_k3 = tl.load(p_k3, mask=m_ck3, other=0.0).to(tl.float32)
b_g3 = tl.load(p_g3, mask=m_ck3, other=0.0).to(tl.float32)
# [BK]
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
@@ -236,82 +227,44 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
# save off-diagonal Aqk blocks and prepare Akk
################################################################################
if i_tc1 < T:
p_Aqk10 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
tl.store(
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_Aqk10 = Aqk + o_c1[:, None] * (HV * BT) + o_i[None, :]
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), mask=m_A1)
p_b1 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
)
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
p_b1 = beta + bos * HV + i_hv + o_c1 * HV
b_b1 = tl.load(p_b1, mask=m_tc1, other=0.0).to(tl.float32)
b_Akk10 = b_Akk10 * b_b1[:, None]
if i_tc2 < T:
p_Aqk20 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Aqk21 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_Aqk20 = Aqk + o_c2[:, None] * (HV * BT) + o_i[None, :]
p_Aqk21 = Aqk + o_c2[:, None] * (HV * BT) + (o_i + BC)[None, :]
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), mask=m_A2)
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), mask=m_A2)
p_b2 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
)
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
p_b2 = beta + bos * HV + i_hv + o_c2 * HV
b_b2 = tl.load(p_b2, mask=m_tc2, other=0.0).to(tl.float32)
b_Akk20 = b_Akk20 * b_b2[:, None]
b_Akk21 = b_Akk21 * b_b2[:, None]
if i_tc3 < T:
p_Aqk30 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Aqk31 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Aqk32 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_Aqk30 = Aqk + o_c3[:, None] * (HV * BT) + o_i[None, :]
p_Aqk31 = Aqk + o_c3[:, None] * (HV * BT) + (o_i + BC)[None, :]
p_Aqk32 = Aqk + o_c3[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
p_b3 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
)
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
p_b3 = beta + bos * HV + i_hv + o_c3 * HV
b_b3 = tl.load(p_b3, mask=m_tc3, other=0.0).to(tl.float32)
b_Akk30 = b_Akk30 * b_b3[:, None]
b_Akk31 = b_Akk31 * b_b3[:, None]
b_Akk32 = b_Akk32 * b_b3[:, None]
p_Akk00 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32)
p_Akk00 = Akkd + o_c0[:, None] * (HV * BC) + o_i[None, :]
p_Akk11 = Akkd + o_c1[:, None] * (HV * BC) + o_i[None, :]
p_Akk22 = Akkd + o_c2[:, None] * (HV * BC) + o_i[None, :]
p_Akk33 = Akkd + o_c3[:, None] * (HV * BC) + o_i[None, :]
b_Ai00 = tl.load(p_Akk00, mask=m_A0, other=0.0).to(tl.float32)
b_Ai11 = tl.load(p_Akk11, mask=m_A1, other=0.0).to(tl.float32)
b_Ai22 = tl.load(p_Akk22, mask=m_A2, other=0.0).to(tl.float32)
b_Ai33 = tl.load(p_Akk33, mask=m_A3, other=0.0).to(tl.float32)
################################################################################
# forward substitution on diagonals
@@ -397,47 +350,27 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
# store full Akk_inv to Akk
################################################################################
p_Akk00 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk10 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
)
p_Akk20 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk21 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
)
p_Akk30 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Akk31 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Akk32 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
)
p_Akk00 = Akk + o_c0[:, None] * (HV * BT) + o_i[None, :]
p_Akk10 = Akk + o_c1[:, None] * (HV * BT) + o_i[None, :]
p_Akk11 = Akk + o_c1[:, None] * (HV * BT) + (o_i + BC)[None, :]
p_Akk20 = Akk + o_c2[:, None] * (HV * BT) + o_i[None, :]
p_Akk21 = Akk + o_c2[:, None] * (HV * BT) + (o_i + BC)[None, :]
p_Akk22 = Akk + o_c2[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
p_Akk30 = Akk + o_c3[:, None] * (HV * BT) + o_i[None, :]
p_Akk31 = Akk + o_c3[:, None] * (HV * BT) + (o_i + BC)[None, :]
p_Akk32 = Akk + o_c3[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
p_Akk33 = Akk + o_c3[:, None] * (HV * BT) + (o_i + 3 * BC)[None, :]
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), mask=m_A0)
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), mask=m_A1)
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), mask=m_A1)
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), mask=m_A2)
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), mask=m_A2)
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), mask=m_A2)
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), mask=m_A3)
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), mask=m_A3)
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), mask=m_A3)
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), mask=m_A3)
@triton.heuristics(
@@ -474,14 +407,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
IS_VARLEN: tl.constexpr,
USE_GATHER: tl.constexpr,
):
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
i_b, i_hv = i_bh // HV, i_bh % HV
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
@@ -505,16 +438,19 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
Aqk = Aqk + (bos * HV + i_hv) * BT
Akk = Akk + (bos * HV + i_hv) * BC
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
o_k = tl.arange(0, BK)
m_k = o_k < K
m_ck = m_c[:, None] & m_k[None, :]
p_q = q + o_c[:, None] * (H * K) + o_k[None, :]
p_k = k + o_c[:, None] * (H * K) + o_k[None, :]
p_g = g + o_c[:, None] * (HV * K) + o_k[None, :]
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
p_beta = beta + o_c * HV
b_q = tl.load(p_q, boundary_check=(0, 1))
b_k = tl.load(p_k, boundary_check=(0, 1))
b_g = tl.load(p_g, boundary_check=(0, 1))
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
b_q = tl.load(p_q, mask=m_ck, other=0.0)
b_k = tl.load(p_k, mask=m_ck, other=0.0)
b_g = tl.load(p_g, mask=m_ck, other=0.0)
b_beta = tl.load(p_beta, mask=m_c, other=0.0).to(tl.float32)
if USE_GATHER:
b_gn = gather(
@@ -546,12 +482,12 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
p_Aqk = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
)
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
m_Aqk_st = m_c[:, None] & (o_i[None, :] < BT)
m_Akk_st = m_c[:, None] & (o_i[None, :] < BC)
p_Aqk = Aqk + o_c[:, None] * (HV * BT) + (i_i * BC + o_i)[None, :]
p_Akk = Akk + o_c[:, None] * (HV * BC) + o_i[None, :]
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_Aqk_st)
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), mask=m_Akk_st)
tl.debug_barrier()
@@ -566,7 +502,7 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
b_Ai += m_I
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), mask=m_Akk_st)
def chunk_kda_fwd_intra(
@@ -108,20 +108,16 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
# g: [B, T, HV, K], beta: [B, T, HV]
p_g = tl.make_block_ptr(
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
p_g = g + i_t * HV * K + o_hv[:, None] * K + o_k[None, :]
p_beta = beta + i_t * HV + o_hv
b_g = tl.load(p_g, mask=m_hk, other=0.0).to(tl.float32)
b_beta = tl.load(p_beta, mask=m_hv, other=0.0).to(tl.float32)
b_k *= b_beta[:, None]
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
p_gj = tl.make_block_ptr(
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
p_gj = g + j * HV * K + o_hv[:, None] * K + o_k[None, :]
b_gj = tl.load(p_gj, mask=m_hk, other=0.0).to(tl.float32)
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
+7 -2
View File
@@ -67,6 +67,8 @@ MSA_TOPK_BLOCKS = 16
MSA_SCORE_TYPE = "max"
MSA_SPARSE_BLOCK_SIZE = 128
MSA_INDEX_HEAD_DIM = 128
# Index head counts the score kernels are instantiated for.
MSA_NUM_INDEX_HEADS = (1, 2)
# Wave width the top-k is written against: it gives one lane per output slot and
# reads the score row in wave-wide strips.
WAVE_SIZE = 64
@@ -181,8 +183,11 @@ def aiter_indexer_unsupported_reason(
f"needs index_head_dim={MSA_INDEX_HEAD_DIM}, "
f"got index_head_dim={index_head_dim}"
)
if num_index_heads > MFMA_COLS:
return f"num_index_heads={num_index_heads} exceeds the {MFMA_COLS} MFMA columns"
if num_index_heads not in MSA_NUM_INDEX_HEADS:
return (
f"needs num_index_heads in {MSA_NUM_INDEX_HEADS}, "
f"got num_index_heads={num_index_heads}"
)
# A decode row's whole query shares one MFMA tile, one column per (token,
# head) pair, so spec decode trades columns against the head count.
if num_index_heads * max_decode_query_len > MFMA_COLS:
@@ -115,15 +115,17 @@ def _index_block_score_kernel(
if BLOCK_SIZE_Q * pid_q >= q_len:
return
q_ptrs = tl.make_block_ptr(
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
shape=(q_len, head_dim),
strides=(stride_q_n, stride_q_d),
offsets=(pid_q * BLOCK_SIZE_Q, 0),
block_shape=(BLOCK_SIZE_Q, head_dim),
order=(1, 0),
o_q = pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)
o_d = tl.arange(0, head_dim)
m_q = o_q < q_len
q_ptrs = (
q_ptr
+ seq_start.to(tl.int64) * stride_q_n
+ pid_h.to(tl.int64) * stride_q_h
+ o_q[:, None].to(tl.int64) * stride_q_n
+ o_d[None, :].to(tl.int64) * stride_q_d
)
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
q = tl.load(q_ptrs, mask=m_q[:, None], other=0.0)
# A matched fp8 pair (e.g. e4m3 x e4m3) lowers to a native FP8 MMA, so
# upcast only when the operands differ: there is no mixed-dtype fp8 MMA,
# including across fp8 flavours (fp8e4nv vs fp8e4b8). bf16 and fp32 loads
@@ -125,7 +125,9 @@ def minimax_m3_use_aiter_sparse_pa(
raise ValueError(
"MiniMax M3 AITER sparse paged attention requires "
f"num_kv_heads == 1 per tensor-parallel rank, got {num_kv_heads}, "
"unless the indexer emits the attend's page table."
"unless the AITER indexer emits the attend's page table. It is not "
"in use here; the 'MiniMax M3 indexer: AITER unavailable' log line "
"says why."
)
return requested
+1 -1
View File
@@ -614,7 +614,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
# With query_fp8, q is emitted only in fp8; the attend dequantizes it
# into ``q`` for the tokens that run on bf16-query kernels.
# into ``q`` for the tokens that run on Triton decode.
query_fp8 = self._allocate_query_fp8(qkv)
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
# the fused kernel emits fp8 directly when this buffer is e4m3.
@@ -23,9 +23,10 @@ _TOPK = 16
# fixed planner allocation used by the MSA decode kernel.
_MAX_QUERY_HEAD_ROWS = 65536
_MAX_DECODE_QUERY_LEN = 32
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
_MIN_CUTLASS_BATCH_SIZE = 16
_DEFAULT_MIN_CUTLASS_BATCH_SIZE = 16
_B300_EAGLE3_QUERY_HEAD_ROWS = 1024
_B300_EAGLE3_MAX_QUERY_LEN = 8
_B300_EAGLE3_HEAD_GEOMETRIES = {(64, 4), (32, 2), (16, 1)}
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
@@ -235,6 +236,22 @@ def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
)
def _min_cutlass_batch_size(
decode_query_len: int,
num_q_heads: int,
num_kv_heads: int,
) -> int:
if (
not current_platform.is_device_capability((10, 3))
or not 1 <= decode_query_len <= _B300_EAGLE3_MAX_QUERY_LEN
or (num_q_heads, num_kv_heads) not in _B300_EAGLE3_HEAD_GEOMETRIES
):
return _DEFAULT_MIN_CUTLASS_BATCH_SIZE
# Use CUTLASS once batch_size * decode_query_len * num_q_heads >= 1024.
query_head_rows = decode_query_len * num_q_heads
return (_B300_EAGLE3_QUERY_HEAD_ROWS + query_head_rows - 1) // query_head_rows
def supports_cutlass_sparse_decode(
*,
decode_backend: MiniMaxM3MSADecodeBackend,
@@ -267,9 +284,14 @@ def should_prepare_decode_metadata(
kv_cache_dtype: str,
page_size: int,
topk_blocks: int,
min_batch_size: int | None = None,
) -> bool:
"""Return whether a graph shape can use the CUTLASS decode path."""
total_q = batch_size * decode_query_len
if min_batch_size is None:
min_batch_size = _min_cutlass_batch_size(
decode_query_len, num_q_heads, num_kv_heads
)
return (
supports_cutlass_sparse_decode(
decode_backend=decode_backend,
@@ -280,7 +302,7 @@ def should_prepare_decode_metadata(
topk_blocks=topk_blocks,
)
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
and (batch_size >= min_batch_size or is_nvfp4_kv_cache(kv_cache_dtype))
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
)
@@ -8,6 +8,7 @@ path for regular decode and speculative verification. NVFP4 KV caches read the
vLLM packed pages directly in both prefill and CUTLASS decode.
"""
import os
from dataclasses import dataclass, replace
from typing import ClassVar
@@ -49,6 +50,11 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
logger = init_logger(__name__)
def _get_msa_cutlass_min_batch_size() -> int | None:
value = os.getenv("VLLM_MINIMAX_M3_MSA_CUTLASS_MIN_BATCH")
return int(value) if value is not None else None
def _dequantize_query(
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
) -> torch.Tensor:
@@ -133,6 +139,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
self.msa_cutlass_min_batch_size = _get_msa_cutlass_min_batch_size()
def build(
self,
@@ -159,6 +166,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
kv_cache_dtype=self.kv_cache_dtype,
page_size=SPARSE_BLOCK_SIZE,
topk_blocks=self.topk_blocks,
min_batch_size=self.msa_cutlass_min_batch_size,
):
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
@@ -266,7 +274,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
q = query[:num_tokens].view(-1, self.num_heads, hd)
out = output[:num_tokens].view(-1, self.num_heads, hd)
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
# decode reads it directly, and the bf16-query kernels read it
# decode and prefill read it directly, and Triton decode reads it
# dequantized into ``query``.
q_fp8 = (
None
@@ -339,11 +347,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
# strided view directly (topK stays innermost-contiguous).
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
qp = (
q[nd:]
if q_fp8 is None
else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:])
)
# An E4M3 query attends as is, its scale folded into softmax_scale.
if q_fp8 is None:
qp, softmax_scale = q[nd:], self.scale
else:
qp, softmax_scale = q_fp8[nd:], self.scale * q_scale_float
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
prefill_topk,
p.cu_seqlens_q,
@@ -375,7 +383,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
max_seqlen_k=p.max_seq_len,
blk_kv=SPARSE_BLOCK_SIZE,
causal=True,
softmax_scale=self.scale,
softmax_scale=softmax_scale,
page_table=p.block_table,
seqused_k=p.seq_lens,
schedule=schedule,
@@ -393,7 +401,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
topK=self.topk_blocks,
blk_kv=SPARSE_BLOCK_SIZE,
causal=True,
softmax_scale=self.scale,
softmax_scale=softmax_scale,
cu_seqlens_q=p.cu_seqlens_q,
cu_seqlens_k=p.cu_seqlens_k,
max_seqlen_q=p.max_query_len,
+6
View File
@@ -32,6 +32,7 @@ from vllm.v1.attention.backends.short_conv_attn import (
from ..common.ngram_embedding import (
Qwen4ExpPLEDeviceEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLENvFp4EmbeddingMethod,
Qwen4ExpPLEPinnedHostEmbedding,
)
from ..common.ops.ple import ple_conv, ple_gate, ple_ngram_ids
@@ -241,6 +242,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
embedding_prefix,
getattr(config, "ple_embedding_dtype", None),
)
if isinstance(embedding_quant_method, Qwen4ExpPLENvFp4EmbeddingMethod):
raise NotImplementedError(
"NVFP4 Qwen4Exp PLE tables are not supported on ROCm yet; use a "
"checkpoint with a BF16 or FP8 PLE table."
)
engram_config = get_current_vllm_config().engram_config
embedding_cls = (
Qwen4ExpPLEPinnedHostEmbedding
+2
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLine
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
@@ -248,6 +249,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
head_size=self.head_dim,
max_position=config.max_position_embeddings,
rope_parameters=config.rope_parameters,
mrope_positions_factor=mrope_positions_factor(vllm_config),
)
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+256 -39
View File
@@ -36,6 +36,11 @@ from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
_e2m1_inline,
_e4m3_inline,
dequantize_to_dtype,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
@@ -70,6 +75,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
reduce_results: bool = True,
) -> None:
del num_ngram_heads, max_total_tokens
super().__init__(
@@ -80,6 +86,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
prefix=prefix,
quant_method=embedding_method,
parallel_group=get_etp_group(),
reduce_results=reduce_results,
)
self.embedding_method = embedding_method
self.data_parallel_rank = data_parallel_rank
@@ -153,14 +160,15 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
return embeddings.narrow(0, slot_offset, local_num_tokens)
@abstractmethod
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Start an asynchronous lookup when supported."""
raise NotImplementedError
@abstractmethod
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Return prefetched embeddings for the full input token set."""
raise NotImplementedError
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
"""Quantization interface shared by device, pinned, and file-gather PLE tables."""
@@ -175,13 +183,28 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
embedding_dtype: str | None = None,
) -> "Qwen4ExpPLEEmbeddingMethod":
"""Select the concrete PLE embedding format for a layer."""
if embedding_dtype == "nvfp4":
return Qwen4ExpPLENvFp4EmbeddingMethod()
if embedding_dtype == "float8_e4m3fn":
return Qwen4ExpPLEFp8EmbeddingMethod()
if embedding_dtype is not None:
raise ValueError(
"ple_embedding_dtype must be 'nvfp4' or 'float8_e4m3fn', "
f"got {embedding_dtype!r}"
)
if quant_config is None:
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
if quant_config._resolve_quant_algo(prefix) == "FP8":
quant_algo = quant_config._resolve_quant_algo(prefix)
if quant_algo == "FP8":
return Qwen4ExpPLEFp8EmbeddingMethod()
if quant_algo == "NVFP4":
return Qwen4ExpPLENvFp4EmbeddingMethod()
if quant_algo is not None:
raise ValueError(
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
f"{prefix!r}; expected FP8 or NVFP4"
)
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(
quant_config, ModelOptQuantConfigBase
@@ -235,6 +258,28 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
return F.embedding(input_, layer.weight)
def lookup_dtype(self, layer: nn.Module) -> torch.dtype:
"""Return the dtype of rows that a pinned-host lookup writes."""
return layer.weight.dtype
def lookup_from_pinned(
self,
layer: "Qwen4ExpPLEPinnedHostEmbedding",
ids: torch.Tensor,
output: torch.Tensor,
) -> None:
"""Copy the ETP-owned rows for ``ids`` from pinned host memory."""
row_bytes = layer.embedding_dim * layer.weight.element_size()
_lookup_ple_embedding_from_pinned_kernel[(ids.numel(),)](
layer._uva_weight,
ids,
output,
row_bytes,
layer.shard_indices.org_vocab_start_index,
layer.shard_indices.org_vocab_end_index,
BLOCK_D=triton.next_power_of_2(row_bytes),
)
def process_weights_after_loading(self, layer: nn.Module) -> None:
"""Bind host-file-gather tables once every checkpoint shard is loaded."""
bind = getattr(layer, "bind_file_shards", None)
@@ -346,6 +391,178 @@ class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
# NVFP4 packs two E2M1 values per byte and shares one E4M3 scale per block.
_NVFP4_VALUES_PER_BYTE = 2
_NVFP4_BLOCK_SIZE = 16
class Qwen4ExpPLENvFp4EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
"""NVFP4 PLE embedding: packed E2M1 rows, block scales, one global scale.
Rows stay packed in storage. A lookup decodes only the selected rows to
the activation dtype, before the ETP reduction.
"""
def create_weights(
self,
layer: Qwen4ExpPLEEmbedding,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size, params_dtype
if input_size_per_partition % _NVFP4_BLOCK_SIZE:
raise ValueError(
"NVFP4 PLE embedding dimension must be divisible by "
f"{_NVFP4_BLOCK_SIZE}, got {input_size_per_partition}"
)
weight_loader = extra_weight_attrs.get("weight_loader")
num_rows = sum(output_partition_sizes)
for name, width, dtype in (
(
"weight",
input_size_per_partition // _NVFP4_VALUES_PER_BYTE,
torch.uint8,
),
(
"weight_scale",
input_size_per_partition // _NVFP4_BLOCK_SIZE,
torch.float8_e4m3fn,
),
):
parameter = ModelWeightParameter(
data=layer.allocate_embedding_weight(num_rows, width, dtype),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter(name, parameter)
weight_scale_2 = create_fp8_scale_parameter(
PerTensorScaleParameter,
output_partition_sizes,
input_size_per_partition,
None,
weight_loader,
scale_dtype=torch.float32,
)
layer.register_parameter("weight_scale_2", weight_scale_2)
if layer.weight_scale.is_pinned():
self._uva_weight_scale = get_accelerator_view_from_cpu_tensor(
layer.weight_scale
)
def process_weights_after_loading(self, layer: nn.Module) -> None:
"""Reject NVFP4 PLE checkpoints without a global scale."""
sentinel = torch.finfo(torch.float32).min
if torch.any(layer.weight_scale_2 == sentinel):
raise ValueError("NVFP4 PLE checkpoint is missing its global scale")
super().process_weights_after_loading(layer)
def lookup_dtype(self, layer: nn.Module) -> torch.dtype:
return layer.params_dtype
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
ids = input_.reshape(-1)
if not input_.is_cuda:
rows = dequantize_to_dtype(
F.embedding(ids, layer.weight),
F.embedding(ids, layer.weight_scale),
layer.weight_scale_2,
layer.params_dtype,
swizzle=False,
)
return rows.reshape(*input_.shape, layer.embedding_dim)
output = torch.empty(
(*input_.shape, layer.embedding_dim),
dtype=layer.params_dtype,
device=input_.device,
)
if ids.numel():
# Device rows are already local, so every ID in range is owned.
_lookup_nvfp4_ple_embedding_kernel[(ids.numel(),)](
layer.weight,
layer.weight_scale.view(torch.uint8),
layer.weight_scale_2,
ids,
output,
layer.embedding_dim,
0,
layer.weight.shape[0],
BLOCK_D=triton.next_power_of_2(layer.embedding_dim),
)
return output
def lookup_from_pinned(
self,
layer: "Qwen4ExpPLEPinnedHostEmbedding",
ids: torch.Tensor,
output: torch.Tensor,
) -> None:
_lookup_nvfp4_ple_embedding_kernel[(ids.numel(),)](
layer._uva_weight,
self._uva_weight_scale.view(torch.uint8),
layer.weight_scale_2,
ids,
output,
layer.embedding_dim,
layer.shard_indices.org_vocab_start_index,
layer.shard_indices.org_vocab_end_index,
BLOCK_D=triton.next_power_of_2(layer.embedding_dim),
)
def dequantize(
self,
layer: nn.Module,
embeddings: torch.Tensor,
output_dtype: torch.dtype,
) -> torch.Tensor:
del layer
# The lookup already decoded the rows to the activation dtype.
return embeddings.to(output_dtype)
@triton.jit
def _lookup_nvfp4_ple_embedding_kernel(
weight_ptr,
scale_ptr,
global_scale_ptr,
ids_ptr,
output_ptr,
embedding_dim,
vocab_start,
vocab_end,
BLOCK_D: tl.constexpr,
):
"""Decode owned NVFP4 PLE rows; write zeros for rows another rank owns."""
row = tl.program_id(0).to(tl.int64)
idx = tl.load(ids_ptr + row).to(tl.int64)
owned = (idx >= vocab_start) & (idx < vocab_end)
local_idx = tl.where(owned, idx - vocab_start, 0)
offsets = tl.arange(0, BLOCK_D)
in_row = offsets < embedding_dim
mask = owned & in_row
packed = tl.load(
weight_ptr + local_idx * (embedding_dim // 2) + offsets // 2,
mask=mask,
other=0,
)
# The low nibble holds the even element and the high nibble the odd one.
codes = (packed >> ((offsets % 2) * 4)) & 0xF
scales = _e4m3_inline(
tl.load(
scale_ptr + local_idx * (embedding_dim // 16) + offsets // 16,
mask=mask,
other=0,
)
)
global_scale = tl.load(global_scale_ptr).to(tl.float32)
values = _e2m1_inline(codes) * scales * global_scale
tl.store(output_ptr + row * embedding_dim + offsets, values, mask=in_row)
class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
"""PLE table allocated on the active model device."""
@@ -358,16 +575,16 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
"""Allocate the complete PLE weight on the active device."""
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Resident embedding prefetch is a no-op."""
return None
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Reject prefetch finalization for resident embeddings."""
raise RuntimeError("Device PLE embedding does not support prefetch")
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
"""Gather ETP inputs, look up embeddings, and select local rows."""
"""Gather ETP IDs, optionally reduce lookup results, and select local rows."""
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
embeddings = super().forward(gathered_ids)
@@ -429,6 +646,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
reduce_results: bool = True,
) -> None:
if not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
@@ -442,10 +660,9 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
num_ngram_heads=num_ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
reduce_results=reduce_results,
)
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
self._row_bytes = self.embedding_dim * self.weight.element_size()
self._block_d = triton.next_power_of_2(self._row_bytes)
self._prefetch_stream: torch.cuda.Stream | None = None
self._prefetch_buffer: torch.Tensor | None = None
self._prefetch_alloc_lock = threading.Lock()
@@ -473,35 +690,28 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
input_ids: torch.Tensor,
output: torch.Tensor | None = None,
) -> torch.Tensor:
"""Look up local ETP rows while preserving the weight storage dtype."""
"""Look up local ETP rows in the embedding method's lookup dtype."""
expected_shape = (*input_ids.shape, self.embedding_dim)
lookup_dtype = self.embedding_method.lookup_dtype(self)
if output is None:
output = torch.empty(
expected_shape,
dtype=self.weight.dtype,
dtype=lookup_dtype,
device=input_ids.device,
)
elif (
tuple(output.shape) != expected_shape
or output.dtype != self.weight.dtype
or output.dtype != lookup_dtype
or output.device != input_ids.device
):
raise ValueError(
"PLE prefetch output must match the input shape, weight dtype, "
"PLE prefetch output must match the input shape, lookup dtype, "
"and input device"
)
flat_ids = input_ids.reshape(-1).long()
if flat_ids.numel():
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
self._uva_weight,
flat_ids,
output,
self._row_bytes,
self.shard_indices.org_vocab_start_index,
self.shard_indices.org_vocab_end_index,
BLOCK_D=self._block_d,
)
self.embedding_method.lookup_from_pinned(self, flat_ids, output)
return output
def sync_lookup(self, ngram_ids: torch.Tensor) -> torch.Tensor:
@@ -528,11 +738,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
return self.parallel_group.all_reduce(embeddings)
@eager_break_during_capture
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
buffer = self._prefetch_buffer
if buffer is None:
@@ -555,7 +761,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
self._prefetch_rows,
self._num_ngram_heads,
self.embedding_dim,
dtype=self.weight.dtype,
dtype=self.embedding_method.lookup_dtype(self),
device=self._uva_weight.device,
)
self._prefetch_buffer = buffer
@@ -581,14 +787,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
"""Join the side stream, reduce ETP shards, and select local rows."""
"""Join the side stream, optionally reduce ETP shards, and select rows."""
prefetch_stream = self._prefetch_stream
if prefetch_stream is None:
raise RuntimeError("pinned PLE finalize requires a prior start_prefetch")
torch.cuda.current_stream().wait_stream(prefetch_stream)
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
embeddings = self._reduce_etp_embeddings(active_output)
embeddings = prefetch_output[: slot_size * self.etp_data_parallel_size]
if self.reduce_results:
embeddings = self._reduce_etp_embeddings(embeddings)
else:
# The outer TP reduce-scatter requires ETP=TP.
assert self.etp_data_parallel_size == 1
embeddings = self._select_embeddings(
embeddings,
output.shape[0],
@@ -596,11 +806,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
)
output.copy_(embeddings.flatten(-2))
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Finish the pinned lookup into graph-owned output storage."""
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
"""Finalize the full-token pinned lookup with optional ETP reduction."""
buffer = self._prefetch_buffer
if buffer is None:
raise RuntimeError("pinned PLE lookup requires a prior start_prefetch")
output = buffer.new_empty((hidden_states.shape[0], self._output_dim))
# total_tokens counts this DP rank's tokens before SP sharding.
output = buffer.new_empty((total_tokens, self._output_dim))
self._finalize_prefetch(buffer, output)
return output
def forward(self, input_: torch.Tensor) -> torch.Tensor:
raise RuntimeError(
"Pinned-host PLE embedding requires "
"start_prefetch() and finalize_prefetch()."
)
+23 -17
View File
@@ -344,6 +344,7 @@ def _ple_conv_kernel(
MODE: tl.constexpr,
HAS_INIT: tl.constexpr,
NULL_STATE_ID: tl.constexpr,
FUSE_RESIDUAL: tl.constexpr,
launch_pdl: tl.constexpr,
):
t = tl.program_id(0)
@@ -438,21 +439,23 @@ def _ple_conv_kernel(
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
y = conv * tl.sigmoid(conv)
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
residual = tl.load(
residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
# Preserve the original eager operation boundaries: short convolution is
# first accumulated into the BF16/FP16 PLE output, then the outer residual
# is added to that rounded value.
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
outer_residual = tl.load(
outer_residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
ple_output = conv_output
if FUSE_RESIDUAL:
residual = tl.load(
residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
# Preserve the original eager operation boundaries: short convolution is
# first accumulated into the BF16/FP16 PLE output, then the outer residual
# is added to that rounded value.
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
outer_residual = tl.load(
outer_residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(
@@ -575,7 +578,7 @@ def ple_conv(
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
state_indices: torch.Tensor,
outer_residual: torch.Tensor,
outer_residual: torch.Tensor | None,
*,
mode: Literal["decode", "spec", "prefill"],
dilation: int,
@@ -585,7 +588,9 @@ def ple_conv(
spec_query_len: int = 1,
token_indices: torch.Tensor | None = None,
) -> None:
"""Add short convolution and the outer residual; update state."""
"""Update state and write activated convolution into residual.
A non-None outer_residual enables both fused residual additions.
"""
BLOCK_C = 512
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
T, C = inputs.shape
@@ -657,6 +662,7 @@ def ple_conv(
SPEC_QUERY_LEN=kernel_spec_query_len,
MODE=mode,
HAS_INIT=has_initial_states_arg,
FUSE_RESIDUAL=outer_residual is not None,
NULL_STATE_ID=NULL_BLOCK_ID,
launch_pdl=launch_pdl,
num_warps=num_warps,
@@ -240,12 +240,20 @@ QWEN4_EXP_SM121_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] =
8: SkinnyGemmConfig(8, 128, 1, vector_width=2, static_k=6144),
16: SkinnyGemmConfig(16, 32, 1, static_k=6144),
},
# QSA fused QKV/gate projection, TP=2.
# QSA fused QKV/gate projection, TP=2 (LoRA keeps the indexer projection separate).
(6656, 2560): {
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
},
# QSA fused QKV/gate + replicated indexer Q/K projection, TP=2.
(7296, 2560): {
1: SkinnyGemmConfig(1, 32, 1, vector_width=2, static_k=2560),
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560),
4: SkinnyGemmConfig(4, 32, 1, vector_width=2, static_k=2560),
8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=2560),
16: SkinnyGemmConfig(16, 32, 4, vector_width=2, static_k=2560),
},
# GDN fused QKVZ projection, TP=2.
(8192, 2560): {
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
+102 -22
View File
@@ -9,8 +9,10 @@ import torch
from torch import nn
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm import envs
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
@@ -37,6 +39,7 @@ from vllm.model_executor.models.interfaces import (
MambaStateShapes,
MixtureOfExperts,
MultiModalEmbeddings,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -68,6 +71,12 @@ from vllm.model_executor.models.utils import (
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
@@ -87,6 +96,14 @@ from .qsa import Qwen4ExpQSAAttention
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
parallel_config = vllm_config.parallel_config
return parallel_config.tensor_parallel_size > 1 and (
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
)
def without_modelopt_fp4(
quant_config: QuantizationConfig | None,
) -> QuantizationConfig | None:
@@ -164,13 +181,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
"""Qwen3Next MoE with Qwen4Exp HC validation."""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
parallel_config = vllm_config.parallel_config
if parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
super().__init__(vllm_config=vllm_config, prefix=prefix)
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__(
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
)
if self.replicate_shared_expert and not self.is_sequence_parallel:
# Reduce routed outputs before adding replicated shared outputs.
# SP then slices the result to avoid summing shared outputs again via RS.
self.experts.moe_config.skip_final_all_reduce = False
config = vllm_config.model_config.hf_text_config
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
@@ -188,11 +211,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
self.config = config
self.layer_type = layer_type
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.layer_idx = extract_layer_index(prefix)
if vllm_config.parallel_config.use_sequence_parallel_moe:
raise NotImplementedError(
"Qwen4Exp HC does not support sequence-parallel MoE"
)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.ple: Qwen4ExpPLELayer | None = None
ple_layer_ids = config.ple_layer_ids
if (self.layer_idx + 1) in ple_layer_ids:
@@ -207,6 +231,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_idx=self.layer_idx,
ple_dense_layer_id=ple_dense_layer_id,
prefix=f"{prefix}.ple",
use_sequence_parallel=self.use_hc_sequence_parallel,
)
if layer_type == "linear_attention":
@@ -215,6 +240,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
vllm_config=vllm_config,
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
reduce_results=not self.use_hc_sequence_parallel,
)
elif layer_type in _QSA_LAYER_TYPES:
use_qsa = getattr(config, "indexer_n_heads", None) is not None
@@ -225,6 +251,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
self.self_attn = Qwen4ExpQSAAttention(
@@ -233,12 +260,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
layer_id=self.layer_idx,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_hc_sequence_parallel,
)
else:
raise ValueError(f"Invalid layer_type {layer_type}")
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
vllm_config=vllm_config,
prefix=f"{prefix}.mlp",
reduce_results=not self.use_hc_sequence_parallel,
)
hc_config = HyperConnectionConfig(
@@ -298,6 +328,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
if self.use_hc_sequence_parallel:
# Positions retain the full token axis for both text and MRoPE inputs.
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in _QSA_LAYER_TYPES:
@@ -309,10 +342,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
raise ValueError("Invalid layer_type")
mlp_hc = self.mlp_hyper_connection
if self.use_hc_sequence_parallel:
attn_out = sp_reduce_scatter(attn_out)
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
hidden_states, attn_out, injection
)
mlp_out = self.mlp(block_input)
if self.use_sequence_parallel:
# MoE consumes local tokens and returns complete local outputs.
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
else:
if self.use_hc_sequence_parallel:
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
mlp_out = self.mlp(block_input)
if self.use_hc_sequence_parallel:
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
# Only sum partial outputs. Reduced outputs already include all ranks.
mlp_out = (
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
)
return hidden_states, mlp_out, injection
@@ -376,6 +424,11 @@ class Qwen4ExpModel(nn.Module):
self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.vocab_size = config.vocab_size
self._qsa_layer_ids = frozenset(
layer_idx
@@ -436,7 +489,6 @@ class Qwen4ExpModel(nn.Module):
@staticmethod
def _start_layer_ple_prefetch(
layer: nn.Module,
hidden_states: torch.Tensor,
input_ids: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
ngram_context: torch.Tensor | None,
@@ -447,12 +499,7 @@ class Qwen4ExpModel(nn.Module):
return
if input_ids is None or query_start_loc is None or ngram_context is None:
raise RuntimeError("PLE inputs were not prepared")
ple.start_prefetch(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
ple.start_prefetch(input_ids, query_start_loc, ngram_context)
def forward(
self,
@@ -471,19 +518,31 @@ class Qwen4ExpModel(nn.Module):
if input_ids is None:
raise ValueError("input_ids or inputs_embeds is required")
hidden_states = self.embed_input_ids(input_ids)
if self.use_hc_sequence_parallel:
if (
self.use_sequence_parallel
and envs.VLLM_MOE_SKIP_PADDING
and is_forward_context_available()
):
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
# Expand only the local token rows when SP is enabled.
hidden_states = hidden_states.repeat(1, self.config.hc_count)
else:
if intermediate_tensors is None:
raise ValueError("pipeline stage requires intermediate tensors")
hidden_states = intermediate_tensors["hidden_states"]
full_num_tokens = positions.shape[-1]
block_output = None
injection = None
last_layer = None
if self.start_layer < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[self.start_layer],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
@@ -495,7 +554,6 @@ class Qwen4ExpModel(nn.Module):
if layer_idx + 1 < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[layer_idx + 1],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
@@ -515,6 +573,8 @@ class Qwen4ExpModel(nn.Module):
deepstack_embed = deepstack_input_embeds[
f"deepstack_input_embeds_{layer_idx}"
]
if self.use_hc_sequence_parallel:
deepstack_embed = sp_shard(deepstack_embed)
deepstack_embed = (
deepstack_embed.unsqueeze(-2)
.expand(
@@ -549,6 +609,24 @@ class Qwen4ExpModel(nn.Module):
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
hidden_states, block_output, injection
)
if self.use_hc_sequence_parallel:
if self._mtp_hidden_buffer is not None:
# Gather LM-head and MTP states together when both are needed.
hidden_size = self.config.hidden_size
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[
:full_num_tokens
]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, self.config.hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
else:
sample_hidden_states = sp_all_gather(sample_hidden_states)[
:full_num_tokens
]
if self._mtp_hidden_buffer is not None:
# Capture the pre-final-mixer multi-stream hidden state
# [T, hc_count*H] for the MTP drafter (zero extra compute:
@@ -604,6 +682,7 @@ class Qwen4ExpModel(nn.Module):
class Qwen4ExpForCausalLM(
nn.Module,
HasInnerState,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
@@ -885,6 +964,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
class Qwen4ExpForConditionalGeneration(
Qwen3_5ForConditionalGeneration,
HasInnerState,
SupportsHCSP,
Qwen4ExpMixtureOfExperts,
SupportsReplaySSM,
):

Some files were not shown because too many files have changed in this diff Show More