Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -68,7 +68,8 @@ class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKe
|
||||
weight_ptr = kv_weight_ptr
|
||||
row_out = kv_out_ptr + token_idx * kv_out_stride
|
||||
|
||||
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
|
||||
# RMSNorm in fp32 throughout — matches
|
||||
# csrc/layernorm_kernels.cu's
|
||||
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
|
||||
# keep x, rrms, and w all in fp32 and perform a single cast at store.
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
|
||||
@@ -179,6 +182,8 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
|
||||
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
@@ -213,6 +218,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
mla_kv_cache = self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
|
||||
kv_c_out = torch.empty_like(kv_c) if use_mha else None
|
||||
k_pe_out = torch.empty_like(k_pe) if use_mha else None
|
||||
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
@@ -237,9 +246,18 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
kv_c_out=kv_c_out,
|
||||
k_pe_out=k_pe_out,
|
||||
)
|
||||
|
||||
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
||||
if use_mha:
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split(
|
||||
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
ql_nope = q_nope
|
||||
else:
|
||||
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
@@ -260,7 +278,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
indexer_n_head_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
quantize_mqa=self._fp8_kv,
|
||||
quantize_mqa=self._fp8_kv and not use_mha,
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
@@ -270,6 +288,19 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
if use_mha:
|
||||
assert kv_c_out is not None and k_pe_out is not None
|
||||
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
|
||||
self.forward_impl(
|
||||
mha_q,
|
||||
kv_c_out,
|
||||
k_pe_out.unsqueeze(1),
|
||||
self.kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
return
|
||||
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
kv_cache = self.kv_cache
|
||||
if self._fp8_kv_needs_view:
|
||||
|
||||
@@ -6,7 +6,9 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
@@ -87,6 +89,7 @@ def _fp8_quant_and_cache_write(
|
||||
SHUFFLE: tl.constexpr,
|
||||
BLOCK_TILE: tl.constexpr,
|
||||
HEAD_TILE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FNUZ: tl.constexpr,
|
||||
):
|
||||
@@ -111,6 +114,8 @@ def _fp8_quant_and_cache_write(
|
||||
|
||||
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
|
||||
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
|
||||
if MX_SCALE:
|
||||
scale = pack_ue8m0_x4(scale)
|
||||
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
|
||||
|
||||
|
||||
@@ -167,6 +172,7 @@ def _fused_norm_rope_kernel(
|
||||
INDEXER_CACHE_SHUFFLE: tl.constexpr,
|
||||
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
|
||||
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
|
||||
INDEXER_CACHE_MX_SCALE: tl.constexpr,
|
||||
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
|
||||
mla_cache_ptr,
|
||||
mla_cache_block_stride,
|
||||
@@ -501,6 +507,7 @@ def _fused_norm_rope_kernel(
|
||||
INDEXER_CACHE_SHUFFLE,
|
||||
INDEXER_CACHE_BLOCK_TILE,
|
||||
INDEXER_CACHE_HEAD_TILE,
|
||||
INDEXER_CACHE_MX_SCALE,
|
||||
FP8_MAX,
|
||||
USE_FNUZ,
|
||||
)
|
||||
@@ -571,7 +578,10 @@ def fused_norm_rope(
|
||||
assert slot_mapping is not None
|
||||
if indexer_slot_mapping is None:
|
||||
indexer_slot_mapping = slot_mapping
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
|
||||
idx_cache_mx_scale = mqa_logits_use_mx_scales()
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(
|
||||
torch.int32 if idx_cache_mx_scale else torch.float32
|
||||
)
|
||||
idx_cache_block_size = indexer_k_cache.shape[1]
|
||||
idx_cache_stride = indexer_k_cache.shape[2]
|
||||
# The caller's cache reports whether its reader expects the shuffled
|
||||
@@ -591,6 +601,7 @@ def fused_norm_rope(
|
||||
)
|
||||
else:
|
||||
idx_cache_scale_view = None
|
||||
idx_cache_mx_scale = False
|
||||
idx_cache_block_size = 1
|
||||
idx_cache_stride = 0
|
||||
idx_cache_shuffle = False
|
||||
@@ -701,6 +712,7 @@ def fused_norm_rope(
|
||||
idx_cache_shuffle,
|
||||
_INDEXER_CACHE_BLOCK_TILE,
|
||||
idx_cache_head_tile,
|
||||
idx_cache_mx_scale,
|
||||
# MLA KV cache (uses same slot_mapping)
|
||||
mla_kv_cache,
|
||||
mla_block_stride,
|
||||
|
||||
@@ -26,10 +26,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -413,12 +409,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -474,11 +464,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
|
||||
@@ -10,8 +10,50 @@ import torch.nn as nn
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_dequantize,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _append_shared_expert_kernel(
|
||||
topk_ids_ptr,
|
||||
topk_weights_ptr,
|
||||
out_ids_ptr,
|
||||
out_weights_ptr,
|
||||
num_tokens,
|
||||
shared_expert_id,
|
||||
TOPK: tl.constexpr,
|
||||
ROUTED_PER_RANK: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
mask = rows < num_tokens
|
||||
for k in tl.static_range(TOPK):
|
||||
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
|
||||
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
|
||||
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
|
||||
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
|
||||
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
|
||||
tl.store(
|
||||
out_ids_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.zeros_like(rows) + shared_expert_id,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
out_weights_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.full([BLOCK_M], 1.0, tl.float32),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
@@ -28,16 +70,25 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
swiglu_limit: float | None,
|
||||
fuse_shared_expert: bool = False,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
ep_group = get_ep_group()
|
||||
self.ep_rank = ep_group.rank_in_group
|
||||
self.ep_size = ep_group.world_size
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_local_experts
|
||||
self.top_k = top_k
|
||||
self.num_routed_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
|
||||
# A fused shared expert is one extra local expert per rank and one
|
||||
# extra route per token, to the token's own rank.
|
||||
self.fuse_shared_expert = fuse_shared_expert
|
||||
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
|
||||
self.num_experts = self.num_local_experts * self.ep_size
|
||||
self.routed_top_k = top_k
|
||||
self.top_k = top_k + fuse_shared_expert
|
||||
self.shared_expert_id = (
|
||||
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.swiglu_limit = swiglu_limit or 0.0
|
||||
@@ -58,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
|
||||
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
|
||||
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
|
||||
self._runtime: Any = None
|
||||
# (max_tok_per_rank, runtime), ascending by capacity.
|
||||
self._runtimes: list[tuple[int, Any]] = []
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -70,7 +122,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_id = expert_id - self.experts_start_idx
|
||||
if not 0 <= local_expert_id < self.num_local_experts:
|
||||
if not 0 <= local_expert_id < self.num_routed_local_experts:
|
||||
return False if return_success else None
|
||||
expert_data = param.data[local_expert_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
@@ -81,8 +133,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
expert_data.copy_(loaded_weight.view(torch.uint8))
|
||||
return True if return_success else None
|
||||
|
||||
def load_shared_expert(self, shared_experts: nn.Module) -> None:
|
||||
slot = self.num_routed_local_experts
|
||||
for linear, weight, weight_scale in (
|
||||
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
|
||||
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
|
||||
):
|
||||
block_scale = get_fp8_block_weight_scale(linear)
|
||||
assert block_scale is not None, "shared expert must be block-FP8"
|
||||
group = GroupShape(
|
||||
linear.weight.shape[0] // block_scale.shape[0],
|
||||
linear.weight.shape[1] // block_scale.shape[1],
|
||||
)
|
||||
packed, scale = mxfp4_quantize(
|
||||
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
|
||||
)
|
||||
weight.data[slot].copy_(packed.view(torch.uint8))
|
||||
weight_scale.data[slot].copy_(scale.view(torch.uint8))
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._runtime is not None:
|
||||
if self._runtimes:
|
||||
return
|
||||
n = self.num_local_experts
|
||||
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
@@ -97,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
|
||||
)
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
FIXED_SLOT_MAX_MTPR,
|
||||
)
|
||||
|
||||
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
|
||||
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
|
||||
# must also be a power of two.
|
||||
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
|
||||
capacities = [self.max_num_tokens]
|
||||
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
|
||||
fixed_slot_max_tokens
|
||||
):
|
||||
capacities.insert(0, fixed_slot_max_tokens)
|
||||
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
|
||||
|
||||
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
build_mega_moe_bundle_plan,
|
||||
)
|
||||
|
||||
return build_mega_moe_bundle_plan(
|
||||
max_tok_per_rank,
|
||||
experts_per_rank=self.num_local_experts,
|
||||
model_dim=self.hidden_size,
|
||||
inter_dim=self.intermediate_size,
|
||||
world_size=self.ep_size,
|
||||
).fixed_slot_dispatch
|
||||
|
||||
def get_runtime(self, max_tok_per_rank: int) -> Any:
|
||||
key = (
|
||||
id(get_ep_group().cpu_group),
|
||||
torch.accelerator.current_device_index(),
|
||||
@@ -104,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.max_num_tokens,
|
||||
max_tok_per_rank,
|
||||
self.swiglu_limit,
|
||||
)
|
||||
runtime = self._runtime_cache.get(key)
|
||||
@@ -123,11 +222,53 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
w1_scale=self.w13_weight_scale,
|
||||
w2=self.w2_weight,
|
||||
w2_scale=self.w2_weight_scale,
|
||||
max_tok_per_rank=self.max_num_tokens,
|
||||
max_tok_per_rank=max_tok_per_rank,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
)
|
||||
self._runtime_cache[key] = runtime
|
||||
self._runtime = runtime
|
||||
return runtime
|
||||
|
||||
def select_runtime(self, num_tokens: int) -> Any:
|
||||
# Dispatch is collective, so every EP rank must pick the same runtime.
|
||||
if is_forward_context_available():
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is not None:
|
||||
num_tokens = max(
|
||||
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
|
||||
)
|
||||
for max_tok_per_rank, runtime in self._runtimes:
|
||||
if num_tokens <= max_tok_per_rank:
|
||||
return runtime
|
||||
raise ValueError(
|
||||
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
|
||||
)
|
||||
|
||||
def append_shared_expert(
|
||||
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Remap routed ids to the per-rank (routed + shared) layout and append
|
||||
a weight-1.0 route to this rank's shared slot."""
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out_ids = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
|
||||
)
|
||||
out_weights = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
|
||||
)
|
||||
if num_tokens:
|
||||
block_m = 64
|
||||
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
|
||||
topk_ids.contiguous(),
|
||||
topk_weights.to(torch.float32).contiguous(),
|
||||
out_ids,
|
||||
out_weights,
|
||||
num_tokens,
|
||||
self.shared_expert_id,
|
||||
TOPK=self.routed_top_k,
|
||||
ROUTED_PER_RANK=self.num_routed_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
)
|
||||
return out_weights, out_ids
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -135,8 +276,10 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
runtime = self._runtime
|
||||
assert runtime is not None
|
||||
assert self._runtimes
|
||||
runtime = self.select_runtime(hidden_states.shape[0])
|
||||
if self.fuse_shared_expert:
|
||||
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
|
||||
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
||||
runtime._s1_w1 = self.w13_weight
|
||||
runtime._s1_w1_scale = self.w13_weight_scale
|
||||
@@ -149,6 +292,10 @@ DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type:
|
||||
|
||||
|
||||
def finalize_mega_moe_weights(model: nn.Module) -> None:
|
||||
for module in model.modules():
|
||||
if isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
for module in list(model.modules()):
|
||||
experts = getattr(module, "experts", None)
|
||||
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
|
||||
if experts.fuse_shared_expert:
|
||||
experts.load_shared_expert(module.shared_experts)
|
||||
module.shared_experts = None
|
||||
experts.finalize_weights()
|
||||
|
||||
@@ -74,11 +74,15 @@ from vllm.model_executor.models.utils import (
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
DeepseekV4AiterMegaMoEExperts,
|
||||
finalize_mega_moe_weights,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
@@ -604,7 +608,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.image_sentinel_lo = (
|
||||
IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0
|
||||
)
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
num_hash_layers = getattr(config, "num_hash_layers", None) # Set by V4.1
|
||||
if num_hash_layers is None:
|
||||
num_hash_layers = config.mlp_layer_types.count("hash_moe")
|
||||
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
@@ -690,6 +697,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
fuse_shared_expert=(
|
||||
config.n_shared_experts == 1
|
||||
and envs.VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS
|
||||
),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
return
|
||||
@@ -1221,7 +1232,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
# Each DBO ubatch owns a different slice of the batch. Writing
|
||||
# from row 0 in both threads would let one ubatch clobber the
|
||||
# other's rows in the buffer the MTP drafter reads after the step.
|
||||
offset = dbo_current_ubatch_token_offset()
|
||||
self._mtp_hidden_buffer[offset : offset + num_tokens].copy_(
|
||||
hidden_states.flatten(1)
|
||||
)
|
||||
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
@@ -1258,14 +1275,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1355,14 +1364,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
@@ -1524,6 +1525,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Must precede the gate_up preshuffle of the fused shared experts.
|
||||
finalize_mega_moe_weights(self)
|
||||
# After per-layer quant finalize, so we preshuffle the final fp8 weights.
|
||||
fused_compressor_layers = 0
|
||||
for module in self.modules():
|
||||
@@ -1532,8 +1535,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
module.prepare_attn_preshuffle()
|
||||
elif isinstance(module, DeepseekV4MLP):
|
||||
module.prepare_gateup_preshuffle()
|
||||
elif isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
if fused_compressor_layers:
|
||||
logger.info(
|
||||
"Fused the C4 compressor GEMMs in %d DeepSeek V4 layers",
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -43,11 +39,12 @@ from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import align_mtp_quant_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .mega_moe import finalize_mega_moe_weights
|
||||
from .model import DeepseekV4DecoderLayer
|
||||
from .model import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -293,10 +290,15 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
# The draft head reuses DeepseekV4DecoderLayer, so its fused linears must
|
||||
# unfold to the same checkpoint shards for per-layer quant config lookup.
|
||||
packed_modules_mapping = DeepseekV4ForCausalLM.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
align_mtp_quant_config(self.quant_config, self.config)
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
@@ -384,14 +386,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
# KeyError downstream rather than be silently rewritten.
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
@@ -489,12 +483,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
|
||||
@@ -162,6 +162,16 @@ def apply_pre_quantized_block_scaled_mm(
|
||||
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
|
||||
|
||||
|
||||
def _default_first_token_threshold() -> int:
|
||||
"""Max tokens at which the CSA/HCA fork enqueues the default chain first.
|
||||
|
||||
At small batches starting the long default chain first shortens the step
|
||||
tail; larger batches saturate the device and aux-first is faster. Only
|
||||
tuned on gfx950 (24 tokens = conc 8 with MTP=2); 0 disables elsewhere.
|
||||
"""
|
||||
return 24 if _ON_GFX950 else 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
@@ -807,6 +817,8 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
assert compressor is not None
|
||||
# Static per captured graph: each capture size bakes its own order.
|
||||
default_first = hidden_states.shape[0] <= _default_first_token_threshold()
|
||||
|
||||
def default_chain():
|
||||
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
|
||||
@@ -838,6 +850,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
[self.ln_events[1]],
|
||||
aux_streams[:1],
|
||||
enable=True,
|
||||
default_first=default_first,
|
||||
)
|
||||
self._sparse_indexer_and_attn(
|
||||
hidden_states, None, None, None, q, kv_out, positions, o_padded
|
||||
@@ -864,6 +877,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
self.ln_events[1:3],
|
||||
aux_streams[:2],
|
||||
enable=True,
|
||||
default_first=default_first,
|
||||
)
|
||||
|
||||
indexer_weights_out, _ = indexer.weights_proj(hidden_states)
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.models.common.ops.fused_qk_rmsnorm import (
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL,
|
||||
fused_q_kv_rmsnorm,
|
||||
@@ -230,7 +231,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# we do this for because MTP layer is not included
|
||||
# in the compress ratio list
|
||||
if layer_id < config.num_hidden_layers:
|
||||
self.compress_ratio = max(1, config.compress_ratios[layer_id])
|
||||
layer_type = config.layer_types[layer_id]
|
||||
self.compress_ratio = max(1, config.compress_rates.get(layer_type, 0))
|
||||
else:
|
||||
self.compress_ratio = 1
|
||||
self.eps = config.rms_norm_eps
|
||||
@@ -240,10 +242,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
|
||||
# Sink padded to the same head count, initialized to -inf (no sink
|
||||
# effect). Weight loading fills the first n_local_heads slots.
|
||||
self.attn_sink = nn.Parameter(
|
||||
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
|
||||
|
||||
self.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared EPLB MoE registration helpers for DeepSeek V4 family models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool:
|
||||
"""Return whether a DSpark draft can share EPLB state with the target.
|
||||
|
||||
Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts
|
||||
use a smaller routed-expert count and cannot share EPLB state.
|
||||
"""
|
||||
return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4"
|
||||
|
||||
|
||||
def collect_moe_layers(
|
||||
moe_model: Any,
|
||||
layers: Iterable[nn.Module],
|
||||
config: object,
|
||||
*,
|
||||
decoder_layer_type: type[nn.Module],
|
||||
moe_type: type[nn.Module],
|
||||
) -> None:
|
||||
"""Populate ``MixtureOfExperts`` fields from decoder layers for EPLB.
|
||||
|
||||
Works for both target backbones and DSpark draft stacks. Draft models pass
|
||||
their shorter ``model.layers`` list; pipeline-parallel placeholder layers
|
||||
are skipped via ``decoder_layer_type``.
|
||||
"""
|
||||
moe_model.num_expert_groups = getattr(config, "n_group", 1)
|
||||
moe_model.moe_layers = []
|
||||
moe_model.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in layers:
|
||||
if not isinstance(layer, decoder_layer_type):
|
||||
continue
|
||||
if not isinstance(layer.ffn, moe_type):
|
||||
continue
|
||||
example_moe = layer.ffn
|
||||
moe_model.moe_mlp_layers.append(layer.ffn)
|
||||
moe_model.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
moe_model.num_moe_layers = len(moe_model.moe_layers)
|
||||
moe_model.extract_moe_parameters(example_moe)
|
||||
@@ -27,7 +27,7 @@ from typing import Any, assert_never
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, DeepseekV4Config
|
||||
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
@@ -48,7 +48,6 @@ from vllm.multimodal.processing.processor import (
|
||||
UpdateMode,
|
||||
_plan_prompt_updates,
|
||||
)
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
|
||||
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
|
||||
COMPRESS_PAD_TO = 4
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -393,7 +394,9 @@ class DequantizeAndGatherKCacheKernel(
|
||||
|
||||
compress_ratios = frozenset(
|
||||
max(1, int(compress_ratio))
|
||||
for compress_ratio in vllm_config.model_config.hf_config.compress_ratios
|
||||
for compress_ratio in get_compress_ratios(
|
||||
vllm_config.model_config.hf_config
|
||||
)
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
@@ -658,7 +661,7 @@ class ComputeGlobalTopkIndicesAndLensKernel(
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
index_topk = vllm_config.model_config.hf_config.index_topk
|
||||
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
|
||||
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
if index_topk <= 0 or cache_block_size <= 0:
|
||||
return []
|
||||
@@ -964,7 +967,7 @@ class CombineTopkSwaIndicesKernel(
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
|
||||
compress_ratios = tuple(
|
||||
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
|
||||
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
|
||||
)
|
||||
max_c128a_topk = (
|
||||
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
|
||||
@@ -1592,7 +1595,7 @@ class BuildFlashinferMixedSparseIndicesKernel(
|
||||
image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0
|
||||
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
|
||||
compress_ratios = tuple(
|
||||
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
|
||||
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
|
||||
)
|
||||
max_c128a_topk = (
|
||||
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
|
||||
|
||||
@@ -33,7 +33,9 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
@@ -742,6 +744,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
TOKEN_STRIDE: tl.constexpr, # 128 for indexer
|
||||
SCALE_DIM: tl.constexpr, # 4 for indexer (1 float32)
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr, # store the scale as 4 packed UE8M0 bytes
|
||||
):
|
||||
"""Fused compress → RMSNorm → RoPE → FP8 quant → store.
|
||||
|
||||
@@ -878,7 +881,10 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
|
||||
# Single float32 scale
|
||||
scale_val = tl.exp2(exponent)
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
|
||||
if MX_SCALE:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.int32)), pack_ue8m0_x4(scale_val))
|
||||
else:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -927,6 +933,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
|
||||
TOKEN_STRIDE: tl.constexpr, # HEAD_SIZE // 2 = 64 packed bytes/token
|
||||
SCALE_DIM: tl.constexpr, # HEAD_SIZE // QUANT_BLOCK = 4 ue8m0 bytes/token
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr, # unused for MXFP4 (kept for signature parity)
|
||||
):
|
||||
"""Fused compress → RMSNorm → RoPE → MXFP4 quant → store.
|
||||
|
||||
@@ -1102,6 +1109,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
block_size: int
|
||||
kv_cache_block_size: int
|
||||
kv_block_stride: int
|
||||
mx_scale: bool
|
||||
|
||||
# Canonical kernel for the launcher's arg_names mapping; the launched
|
||||
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
|
||||
@@ -1181,6 +1189,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
block_size=state_block_size,
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
kv_block_stride=kv_block_stride,
|
||||
mx_scale=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
@@ -1301,6 +1310,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
TOKEN_STRIDE=token_stride,
|
||||
SCALE_DIM=scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
num_warps=1,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
|
||||
from .mm_preprocess import (
|
||||
COMPRESS_PAD_TO,
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
@@ -105,6 +106,10 @@ class DeepseekV4ForConditionalGeneration(
|
||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
supports_tower_connector_lora = True
|
||||
# Image spans widen the sparse-SWA window in-kernel and the placeholder
|
||||
# starts with a compressor-alignment pad
|
||||
mm_prefix_clamp_sliding_window = True
|
||||
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, cast
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -409,12 +409,12 @@ class DeepseekCompressor(nn.Module):
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
)
|
||||
|
||||
# full graph cannot branch on per-step CPU metadata after capture
|
||||
# A captured graph would replay the capture batch's skip decision.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and self.head_dim == 512
|
||||
and self.compress_ratio == 128
|
||||
and forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
and state_metadata.c128_boundary is False
|
||||
):
|
||||
return
|
||||
|
||||
@@ -145,7 +145,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
|
||||
"hash_moe"
|
||||
)
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
@@ -663,14 +665,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -755,14 +749,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -18,10 +18,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -46,17 +42,20 @@ from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
DeepseekV4Model,
|
||||
DeepseekV4MoE,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
@@ -346,7 +345,7 @@ def _insert_context_kv(
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
@@ -373,6 +372,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
@@ -451,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=first_layer.ffn.n_redundant_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
@@ -470,12 +480,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
@@ -493,6 +497,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts." in name:
|
||||
experts = self.model.layers[extract_layer_index(name)].ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
@@ -544,11 +555,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
@@ -560,17 +566,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
self.process_weights_after_loading()
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def _finalize_moe(self) -> None:
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self._finalize_moe()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
@@ -33,9 +32,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
DeepGemmMegaMoEBackend,
|
||||
get_deep_gemm_mega_moe_backend,
|
||||
ue8m0_uint8_to_float,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
eplb_map_to_physical_and_record,
|
||||
@@ -81,6 +78,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -224,6 +222,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_shared_experts: int = 0,
|
||||
shared_experts: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
num_logical_experts: int | None = None,
|
||||
):
|
||||
@@ -237,7 +236,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
self.num_logical_experts = (
|
||||
@@ -246,64 +244,33 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
|
||||
self.eplb_state = EplbLayerState()
|
||||
|
||||
weight_attrs = {"weight_loader": self.weight_loader}
|
||||
self.w13_weight = nn.Parameter(
|
||||
torch.zeros(
|
||||
num_local_experts,
|
||||
2 * intermediate_size,
|
||||
hidden_size // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
device = torch.get_default_device()
|
||||
if device.type != current_platform.device_type: # meta under ipc_cache
|
||||
device = torch.device(
|
||||
current_platform.device_type, torch.accelerator.current_device_index()
|
||||
)
|
||||
self.backend = get_deep_gemm_mega_moe_backend(
|
||||
device, hidden_size, intermediate_size
|
||||
)
|
||||
set_weight_attrs(self.w13_weight, weight_attrs)
|
||||
|
||||
self.w13_weight_scale = nn.Parameter(
|
||||
torch.zeros(
|
||||
num_local_experts,
|
||||
2 * intermediate_size,
|
||||
hidden_size // 32,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
if num_shared_experts and not self.backend.supports_shared_experts(
|
||||
shared_experts
|
||||
):
|
||||
num_shared_experts = 0
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.backend.create_weights(
|
||||
self,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_shared_experts=num_shared_experts,
|
||||
)
|
||||
set_weight_attrs(self.w13_weight_scale, weight_attrs)
|
||||
for name, param in self.named_parameters(recurse=False):
|
||||
shared = name.startswith("shared_")
|
||||
loader = self.shared_weight_loader if shared else self.weight_loader
|
||||
set_weight_attrs(param, {"weight_loader": loader})
|
||||
self.w13_weight_scale.quant_method = "block"
|
||||
|
||||
self.w2_weight = nn.Parameter(
|
||||
torch.zeros(
|
||||
num_local_experts,
|
||||
hidden_size,
|
||||
intermediate_size // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(self.w2_weight, weight_attrs)
|
||||
|
||||
self.w2_weight_scale = nn.Parameter(
|
||||
torch.zeros(
|
||||
num_local_experts,
|
||||
hidden_size,
|
||||
intermediate_size // 32,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(self.w2_weight_scale, weight_attrs)
|
||||
self.w2_weight_scale.quant_method = "block"
|
||||
|
||||
self._transformed_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._transformed_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._transformed_shared_l1_weights: (
|
||||
tuple[torch.Tensor, torch.Tensor] | None
|
||||
) = None
|
||||
self._transformed_shared_l2_weights: (
|
||||
tuple[torch.Tensor, torch.Tensor] | None
|
||||
) = None
|
||||
|
||||
self._backend: DeepGemmMegaMoEBackend | None = None
|
||||
|
||||
# Register in the static forward context so the custom-op wrapper
|
||||
# can look up this module by name from within a torch.compile graph.
|
||||
compilation_config = vllm_config.compilation_config
|
||||
@@ -331,144 +298,57 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_ids = self._map_global_expert_id(expert_id)
|
||||
if not local_expert_ids:
|
||||
param_prefix = "w2_" if shard_id == "w2" else "w13_"
|
||||
if not local_expert_ids or param_prefix not in weight_name:
|
||||
return False if return_success else None
|
||||
|
||||
loaded_any = False
|
||||
is_scale = weight_name.endswith("weight_scale")
|
||||
for local_expert_id in local_expert_ids:
|
||||
expert_data = param.data[local_expert_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
if "w13_" not in weight_name:
|
||||
continue
|
||||
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
|
||||
expert_data = expert_data.narrow(
|
||||
0, shard_offset, self.intermediate_size
|
||||
)
|
||||
elif shard_id == "w2":
|
||||
if "w2_" not in weight_name:
|
||||
continue
|
||||
else:
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
|
||||
if expert_data.shape != loaded_weight.shape:
|
||||
raise ValueError(
|
||||
f"DeepSeek V4 MegaMoE expert weight shape mismatch for "
|
||||
f"{weight_name}: parameter shard {tuple(expert_data.shape)} "
|
||||
f"vs checkpoint {tuple(loaded_weight.shape)}"
|
||||
)
|
||||
expert_data.copy_(loaded_weight)
|
||||
loaded_any = True
|
||||
|
||||
if return_success:
|
||||
return loaded_any
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
|
||||
return ue8m0_uint8_to_float(sf)
|
||||
|
||||
def _ensure_backend(self) -> DeepGemmMegaMoEBackend:
|
||||
"""Lazily resolve the MegaMoE backend.
|
||||
|
||||
The backend is looked up via :func:`get_deep_gemm_mega_moe_backend`, which
|
||||
performs all runtime capability checks. Failure propagates directly
|
||||
to the caller. This must be called before :meth:`finalize_weights`
|
||||
releases the loader-side parameters.
|
||||
"""
|
||||
if self._backend is not None:
|
||||
return self._backend
|
||||
assert self.w13_weight is not None, (
|
||||
"_ensure_backend() must be called before finalize_weights() "
|
||||
"releases the loader-side parameters."
|
||||
)
|
||||
self._backend = get_deep_gemm_mega_moe_backend(
|
||||
self.w13_weight.device,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
)
|
||||
return self._backend
|
||||
|
||||
def _check_runtime_supported(self) -> None:
|
||||
"""Preserved for API compatibility; delegates to the backend check."""
|
||||
self._ensure_backend()
|
||||
|
||||
def _finalize_shared_expert_weights(self, shared_experts: DeepseekV4MLP) -> None:
|
||||
"""Delegate shared-expert weight transform to the backend and update state.
|
||||
|
||||
On success, populates ``self._transformed_shared_l{1,2}_weights``. If
|
||||
the backend cannot fuse shared experts (returns ``None``), disables
|
||||
native fusion by setting ``self.num_shared_experts = 0``.
|
||||
"""
|
||||
backend = self._ensure_backend()
|
||||
result = backend.transform_shared_expert_weights(
|
||||
shared_experts=shared_experts,
|
||||
num_shared_experts=self.num_shared_experts,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
prefix=self.prefix,
|
||||
)
|
||||
if result is None:
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
(
|
||||
self._transformed_shared_l1_weights,
|
||||
self._transformed_shared_l2_weights,
|
||||
) = result
|
||||
|
||||
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
|
||||
backend = self._ensure_backend()
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
if self._transformed_l1_weights is None:
|
||||
self._transformed_l1_weights, self._transformed_l2_weights = (
|
||||
backend.transform_weights(
|
||||
w13_weight=self.w13_weight.data,
|
||||
w13_weight_scale=self.w13_weight_scale.data,
|
||||
w2_weight=self.w2_weight.data,
|
||||
w2_weight_scale=self.w2_weight_scale.data,
|
||||
num_local_experts=self.num_local_experts,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
)
|
||||
self.backend.load_weight(
|
||||
param.data[local_expert_id], loaded_weight, shard_id, is_scale
|
||||
)
|
||||
# Drop the original loader-side parameters: the MegaMoE kernels only
|
||||
# consume the transformed views above. transform_weights_for_mega_moe
|
||||
# allocates a fresh tensor for the L1 weight (see
|
||||
# _interleave_l1_weights) and fresh SF tensors for L1/L2; the L2
|
||||
# weight is the only tensor that aliases the original storage, and
|
||||
# _transformed_l2_weights still holds it, so the storage stays live
|
||||
# after we drop the Parameter.
|
||||
self.w13_weight = None
|
||||
self.w13_weight_scale = None
|
||||
self.w2_weight = None
|
||||
self.w2_weight_scale = None
|
||||
return True if return_success else None
|
||||
|
||||
if shared_experts is None or self.num_shared_experts == 0:
|
||||
return
|
||||
if self._transformed_shared_l1_weights is not None:
|
||||
return
|
||||
if not backend.supports_shared_experts(deep_gemm):
|
||||
logger.warning_once(
|
||||
"Disabling native MegaMoE shared-expert fusion because the "
|
||||
"installed DeepGEMM Python API is older than the vLLM "
|
||||
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
|
||||
)
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
|
||||
"""Load one shared-expert checkpoint tensor; return the parameter name."""
|
||||
if ".w2." in name or ".down_proj." in name:
|
||||
param_name = "shared_l2_weight"
|
||||
elif ".w1." in name or ".w3." in name:
|
||||
param_name = "shared_l1_weight"
|
||||
else:
|
||||
raise ValueError(f"Unexpected shared-expert tensor: {name}")
|
||||
if "weight_scale" in name:
|
||||
param_name += "_scale"
|
||||
param = getattr(self, param_name)
|
||||
param.weight_loader(param, loaded_weight, name)
|
||||
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
|
||||
|
||||
self._finalize_shared_expert_weights(shared_experts)
|
||||
def shared_weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
|
||||
) -> None:
|
||||
shard_id = (
|
||||
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
|
||||
)
|
||||
is_scale = "weight_scale" in weight_name
|
||||
self.backend.load_weight(param.data, loaded_weight, shard_id, is_scale)
|
||||
|
||||
@property
|
||||
def _transformed_l1_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.w13_weight, self.w13_weight_scale
|
||||
|
||||
@property
|
||||
def _transformed_l2_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.w2_weight, self.w2_weight_scale
|
||||
|
||||
@property
|
||||
def has_fused_shared_experts(self) -> bool:
|
||||
return self._transformed_shared_l1_weights is not None
|
||||
return self.num_shared_experts > 0
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
backend = self._ensure_backend()
|
||||
backend = self.backend
|
||||
group = get_ep_group().device_group
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (
|
||||
@@ -514,10 +394,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
)
|
||||
|
||||
def get_expert_weights(self) -> list[torch.Tensor]:
|
||||
self.finalize_weights()
|
||||
assert self._transformed_l1_weights is not None
|
||||
assert self._transformed_l2_weights is not None
|
||||
|
||||
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
|
||||
assert t.shape[0] == self.num_local_experts
|
||||
@@ -600,7 +476,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
backend = self._ensure_backend()
|
||||
backend = self.backend
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
shared_x_sf = None
|
||||
shared_block_m = None
|
||||
@@ -629,28 +505,21 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
hidden_quant=backend.hidden_quant,
|
||||
)
|
||||
|
||||
assert self._transformed_l1_weights is not None
|
||||
assert self._transformed_l2_weights is not None
|
||||
if self.has_fused_shared_experts:
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
shared_l1_weights=self._transformed_shared_l1_weights,
|
||||
shared_l2_weights=self._transformed_shared_l2_weights,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
else:
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
fused = self.has_fused_shared_experts
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
shared_l1_weights=(
|
||||
(self.shared_l1_weight, self.shared_l1_weight_scale) if fused else None
|
||||
),
|
||||
shared_l2_weights=(
|
||||
(self.shared_l2_weight, self.shared_l2_weight_scale) if fused else None
|
||||
),
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
@@ -841,9 +710,12 @@ class DeepseekV4MoE(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
|
||||
shared_experts=self.shared_experts,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
|
||||
if self.experts.has_fused_shared_experts:
|
||||
self.shared_experts = None
|
||||
|
||||
def _init_fused_moe_experts(
|
||||
self,
|
||||
@@ -880,6 +752,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
ckpt_names=("w1", "w2", "w3"),
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
@@ -910,12 +783,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
return self._forward_fused_moe(hidden_states, input_ids)
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
# The MegaMoE experts module owns the backend; resolve and assert it.
|
||||
self.experts._ensure_backend()
|
||||
assert self.experts._backend is not None, (
|
||||
"MegaMoE backend must be resolved before routing."
|
||||
)
|
||||
mega_backend = self.experts._backend
|
||||
mega_backend = self.experts.backend
|
||||
# Small local padded batches favor GateLinear; 128-expert gates cross earlier.
|
||||
gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16
|
||||
if (
|
||||
@@ -982,10 +850,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
activation_clamp=activation_clamp,
|
||||
)
|
||||
|
||||
if (
|
||||
self.shared_experts is not None
|
||||
and not self.experts.has_fused_shared_experts
|
||||
):
|
||||
if self.shared_experts is not None:
|
||||
shared_output = self.shared_experts(hidden_states)
|
||||
final_hidden_states += shared_output
|
||||
|
||||
@@ -1003,10 +868,6 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
if self.use_native_mega_moe:
|
||||
self.experts.finalize_weights(self.shared_experts)
|
||||
|
||||
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
"""Pick the CUDA sparse-MLA attention class for the configured backend.
|
||||
@@ -1083,7 +944,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
num_hash_layers=config.num_hash_layers,
|
||||
num_hash_layers=config.mlp_layer_types.count("hash_moe"),
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -1346,7 +1207,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.has_local_hash_moe = self.start_layer < min(
|
||||
self.end_layer, config.num_hash_layers
|
||||
self.end_layer, config.mlp_layer_types.count("hash_moe")
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
@@ -1538,14 +1399,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1558,6 +1411,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if ".shared_experts." in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
experts = self.layers[extract_layer_index(name)].ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = self._pad_shared_expert_weight(
|
||||
self.quant_config, name, loaded_weight
|
||||
@@ -1620,14 +1482,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
@@ -1687,10 +1541,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def finalize_mhc_broadcast_weights(self) -> None:
|
||||
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
|
||||
return
|
||||
@@ -1841,23 +1691,13 @@ class DeepseekV4ForCausalLM(
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.num_moe_layers = self.config.num_hidden_layers
|
||||
self.moe_layers: list[nn.Module] = []
|
||||
self.moe_mlp_layers: list[DeepseekV4MoE] = []
|
||||
example_moe: DeepseekV4MoE | None = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if not isinstance(layer, DeepseekV4DecoderLayer):
|
||||
continue
|
||||
if isinstance(layer.ffn, DeepseekV4MoE):
|
||||
example_moe = layer.ffn
|
||||
self.moe_mlp_layers.append(layer.ffn)
|
||||
self.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -1900,7 +1740,6 @@ class DeepseekV4ForCausalLM(
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model.finalize_mega_moe_weights()
|
||||
self.model.finalize_mhc_broadcast_weights()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
|
||||
@@ -21,10 +21,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -386,14 +382,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_native_mega_moe:
|
||||
@@ -436,6 +424,13 @@ class DeepSeekV4MTP(nn.Module):
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts." in name:
|
||||
experts = self.model.layers[str(spec_layer)].mtp_block.ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
@@ -490,12 +485,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
# ``e_score_correction_bias`` lives on the gate
|
||||
@@ -529,17 +518,9 @@ class DeepSeekV4MTP(nn.Module):
|
||||
f"Use a checkpoint that includes MTP layer weights, "
|
||||
f"or disable speculative decoding."
|
||||
)
|
||||
self.process_weights_after_loading()
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for layer in self.model.layers.values():
|
||||
layer.mtp_block.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.finalize_mega_moe_weights()
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
|
||||
|
||||
class DequantGatherKCacheKernel(
|
||||
@@ -329,7 +330,7 @@ class DequantGatherKCacheKernel(
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
|
||||
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
dict(
|
||||
|
||||
@@ -436,7 +436,7 @@ class SparseAttnCompressNormRopeStoreC4Kernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 4 in hf_config.compress_ratios
|
||||
and "compressed_sparse_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype == "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
@@ -934,7 +934,7 @@ class SparseAttnCompressNormRopeStoreFullC4Kernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 4 in hf_config.compress_ratios
|
||||
and "compressed_sparse_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype != "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
@@ -1366,7 +1366,7 @@ class SparseAttnCompressC128Block8Kernel(
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_size=hf_config.head_dim,
|
||||
state_width=hf_config.head_dim,
|
||||
enabled=128 in hf_config.compress_ratios,
|
||||
enabled="heavily_compressed_attention" in hf_config.layer_types,
|
||||
_when=lambda *, enabled: enabled,
|
||||
)
|
||||
|
||||
@@ -2040,7 +2040,7 @@ class SparseAttnNormRopeStoreFullKernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 128 in hf_config.compress_ratios
|
||||
and "heavily_compressed_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype != "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
|
||||
@@ -21,11 +21,41 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
|
||||
|
||||
def align_mtp_quant_config(
|
||||
quant_config: QuantizationConfig | None, config: PreTrainedConfig
|
||||
) -> None:
|
||||
"""Re-key the quant config's MTP entries onto the draft module prefixes.
|
||||
|
||||
``_make_deepseek_v4_weights_mapper`` renames the checkpoint's ``mtp.{i}.``
|
||||
layers to ``model.mtp.{i}.``, but the draft modules are built under
|
||||
``model.layers.{num_hidden_layers + i}.``. Without this remap no per-layer
|
||||
entry matches, so every MTP linear falls back to the global scheme. On a
|
||||
mixed checkpoint (FP8 attention and shared experts, NVFP4 routed experts)
|
||||
that builds NVFP4 parameters for FP8 weights and the load then fails a
|
||||
shape assertion.
|
||||
"""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
if quant_config is None:
|
||||
return
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
f"model.mtp.{i}.": f"model.layers.{config.num_hidden_layers + i}."
|
||||
for i in range(config.num_nextn_predict_layers)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FP8Config(Fp8Config):
|
||||
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
|
||||
|
||||
|
||||
@@ -15,10 +15,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -322,12 +318,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -383,13 +373,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
|
||||
@@ -638,7 +638,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
|
||||
"hash_moe"
|
||||
)
|
||||
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
@@ -1149,14 +1151,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1219,14 +1213,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -335,14 +331,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_mega_moe:
|
||||
@@ -441,12 +429,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
|
||||
@@ -88,10 +88,13 @@ def _xpu_qnorm_rope_kernel(
|
||||
kv_base = kv_ptr + token_idx * HEAD_DIM
|
||||
kv_out_base = kv_out_ptr + token_idx * HEAD_DIM
|
||||
|
||||
# Copy full KV unchanged first
|
||||
# Copy only the NoPE portion; the RoPE region is written exclusively
|
||||
# by the rotated stores below. Masking avoids a write-after-write
|
||||
# hazard on the overlapping RoPE addresses (matches the Q branch).
|
||||
offs = tl.arange(0, HEAD_DIM)
|
||||
kv_full = tl.load(kv_base + offs)
|
||||
tl.store(kv_out_base + offs, kv_full)
|
||||
nope_mask = offs < NOPE_DIM
|
||||
tl.store(kv_out_base + offs, kv_full, mask=nope_mask)
|
||||
|
||||
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions
|
||||
even_offs = NOPE_DIM + rope_pair_idx * 2
|
||||
|
||||
@@ -18,10 +18,6 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
|
||||
@@ -420,12 +416,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -487,11 +477,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
|
||||
@@ -12,8 +12,8 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
@@ -58,8 +58,10 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
from vllm.models.deepseek_v4.amd.model import (
|
||||
DeepseekV4MoE as DeepseekV4MoEBase,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono_decode import MonoDecodeLayer
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -255,6 +257,12 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# attn_norm / ffn_norm into its collapse, so the separate norms are
|
||||
# skipped for the seams it takes.
|
||||
self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
# Decode steps of an eligible layer may run as the mono decode layer.
|
||||
self.mono = MonoDecodeLayer.create(self, vllm_config)
|
||||
if self.mono is not None and self.mono.ffn_only:
|
||||
# A decode step hands wo_b's TP partial to the FFN launch; forward
|
||||
# reduces it on the others.
|
||||
self.attn.wo_b.reduce_results = False
|
||||
|
||||
@staticmethod
|
||||
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
|
||||
@@ -273,6 +281,20 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if self.mono is not None:
|
||||
out = self.mono(
|
||||
self,
|
||||
x,
|
||||
positions,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
)
|
||||
if out is not None:
|
||||
return out
|
||||
# Layer 0's attention seam projects the 2-D embedding with the folded
|
||||
# hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn.
|
||||
# The fused kernel only takes the latter, so that seam keeps the
|
||||
@@ -371,6 +393,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x = self.attn(positions, x, None)
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
elif self.mono is not None and self.mono.ffn_only:
|
||||
out = self.mono.ffn(self, x, residual, post_mix, res_mix, attn_pre)
|
||||
if out is not None:
|
||||
return out
|
||||
x = tensor_model_parallel_all_reduce(x)
|
||||
|
||||
residual, post_mix, res_mix, x, ffn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
@@ -394,6 +421,42 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix, ffn_pre
|
||||
|
||||
def write_kv(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor,
|
||||
post_mix: torch.Tensor,
|
||||
res_mix: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write the KV ``forward`` would for the replay layers' inputs, skipping
|
||||
the rest of the layer; the replay batch then reruns its rows."""
|
||||
# The replay batch rules out Engram and sequence parallel here.
|
||||
assert self.engram is None and not self.use_sequence_parallel
|
||||
fuse_attn_norm = self.fuse_seam_norm
|
||||
residual, post_mix, res_mix, x, _attn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
pre_mix=pre_mix,
|
||||
sublayer_out=x,
|
||||
post_layer_mix=post_mix,
|
||||
comb_res_mix=res_mix,
|
||||
norm_weight=self.attn_norm.weight if fuse_attn_norm else None,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
if not fuse_attn_norm:
|
||||
x = self.attn_norm(x)
|
||||
self.attn.forward_kv(positions, x)
|
||||
|
||||
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
@@ -471,6 +534,45 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
# Config flag is the switch: ROCm forces swa_cache_layer.bounded_replay off.
|
||||
self.decoder_replay_layers: DecoderReplayLayers | None = None
|
||||
self.decoder_replay_start = self.end_layer
|
||||
cut = max(config.kv_source_layer_ids)
|
||||
if (
|
||||
vllm_config.cache_config.swa_bounded_replay
|
||||
and vllm_config.use_v2_model_runner
|
||||
and cut < self.end_layer - 1
|
||||
and self._decoder_replay_supported(vllm_config, cut)
|
||||
):
|
||||
assert self.layers[cut].attn.compress_ratio <= 1
|
||||
# The last KV source writes every row's KV, then replays.
|
||||
self.decoder_replay_start = cut
|
||||
# The attention metadata keys the replay layers read.
|
||||
metadata_prefixes: set[str] = set()
|
||||
for layer in islice(self.layers, cut, self.end_layer):
|
||||
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
|
||||
metadata_prefixes.add(attn.swa_cache_layer.prefix)
|
||||
if attn.compressed_cache_prefix is not None:
|
||||
metadata_prefixes.add(attn.compressed_cache_prefix)
|
||||
if attn.indexer is not None:
|
||||
metadata_prefixes.add(attn.indexer.k_cache.prefix)
|
||||
self.decoder_replay_layers = DecoderReplayLayers(
|
||||
config.sliding_window,
|
||||
self._run_replay_layers,
|
||||
self.layers[cut].write_kv,
|
||||
metadata_prefixes,
|
||||
self.layers[cut].attn.swa_cache_layer.prefix,
|
||||
)
|
||||
logger.info_once(
|
||||
"Decoder SWA bounded replay: in eager prefill steps, layer %d "
|
||||
"writes its KV for every token, and the rest of it and layers "
|
||||
"%d-%d run on each request's last %d tokens only.",
|
||||
cut,
|
||||
cut + 1,
|
||||
self.end_layer - 1,
|
||||
config.sliding_window,
|
||||
)
|
||||
|
||||
# The n-gram hash needs a slot-keyed rolling store of compressed ids
|
||||
# (chunked prefill / decode lookback); key it off the first local
|
||||
# layer's sliding-window KV cache. Only PP ranks owning an engram
|
||||
@@ -630,13 +732,33 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if not get_pp_group().is_first_rank:
|
||||
assert intermediate_tensors is not None
|
||||
pre_mix = intermediate_tensors["pre_mix"]
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
|
||||
(
|
||||
hidden_states,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
aux_hidden_states,
|
||||
final_aux_recon,
|
||||
ran_layers,
|
||||
) = self._run_layers(
|
||||
range(self.start_layer, self.decoder_replay_start),
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
full_num_tokens,
|
||||
)
|
||||
# Replay collapses inside its own batch. The early loop still sets
|
||||
# ran_layers, so that flag alone must not post again on the full batch.
|
||||
collapsed = False
|
||||
if self.decoder_replay_layers is not None:
|
||||
hidden_states, pre_mix, *late_aux = self.decoder_replay_layers(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
@@ -644,20 +766,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
)
|
||||
if idx + 1 in self.aux_hidden_state_layers:
|
||||
# Reconstruct the aux hidden state for draft models
|
||||
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
|
||||
aux_hidden_state = aux_recon.mean(dim=1)
|
||||
if self.use_sequence_parallel:
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
final_aux_recon = aux_recon
|
||||
if layer is not None:
|
||||
# Reuse if the last layer was captured as an aux hidden state
|
||||
if self.end_layer in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.extend(late_aux)
|
||||
final_aux_recon = None
|
||||
collapsed = True
|
||||
if ran_layers and not collapsed:
|
||||
if (
|
||||
self.end_layer in self.aux_hidden_state_layers
|
||||
and final_aux_recon is not None
|
||||
):
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = self.mhc_post(
|
||||
@@ -687,6 +804,162 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def _run_layers(
|
||||
self,
|
||||
layer_ids: range,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
residual: torch.Tensor | None,
|
||||
engram_hashes: torch.Tensor | None,
|
||||
engram_mask: torch.Tensor | None,
|
||||
full_num_tokens: int,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
list[torch.Tensor],
|
||||
torch.Tensor | None,
|
||||
bool,
|
||||
]:
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None
|
||||
ran_layers = False
|
||||
for idx in layer_ids:
|
||||
ran_layers = True
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix = self.layers[idx](
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
)
|
||||
# The aux id of the first replay layer is the post of the last
|
||||
# full-batch layer. Capture it on the trimmed rows instead.
|
||||
if idx + 1 in self.aux_hidden_state_layers and not (
|
||||
self.decoder_replay_layers is not None
|
||||
and idx + 1 == self.decoder_replay_start
|
||||
):
|
||||
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
|
||||
aux_hidden_state = aux_recon.mean(dim=1)
|
||||
if self.use_sequence_parallel:
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
final_aux_recon = aux_recon
|
||||
return (
|
||||
hidden_states,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
aux_hidden_states,
|
||||
final_aux_recon,
|
||||
ran_layers,
|
||||
)
|
||||
|
||||
def _run_replay_layers(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor,
|
||||
post_mix: torch.Tensor,
|
||||
res_mix: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Layers past the last KV source, on the rows they are given.
|
||||
|
||||
The final ``mhc_post`` runs here, on this batch. The return is
|
||||
``(hidden_states, pre_mix, *aux)``.
|
||||
"""
|
||||
# Post of the layer before the cut. These rows are already trimmed,
|
||||
# and sequence parallel is unsupported while replay is on.
|
||||
cut_aux: list[torch.Tensor] = []
|
||||
if self.decoder_replay_start in self.aux_hidden_state_layers:
|
||||
cut_aux.append(
|
||||
self.mhc_post(hidden_states, residual, post_mix, res_mix).mean(dim=1)
|
||||
)
|
||||
(
|
||||
hidden_states,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
aux,
|
||||
final_aux_recon,
|
||||
ran_layers,
|
||||
) = self._run_layers(
|
||||
range(self.decoder_replay_start, self.end_layer),
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
None,
|
||||
None,
|
||||
positions.shape[0],
|
||||
)
|
||||
assert residual is not None and post_mix is not None
|
||||
assert res_mix is not None and pre_mix is not None
|
||||
if (
|
||||
self.end_layer in self.aux_hidden_state_layers
|
||||
and final_aux_recon is not None
|
||||
):
|
||||
hidden_states = final_aux_recon
|
||||
elif ran_layers:
|
||||
hidden_states = self.mhc_post(hidden_states, residual, post_mix, res_mix)
|
||||
return (hidden_states, pre_mix, *cut_aux, *aux)
|
||||
|
||||
def _decoder_replay_supported(self, vllm_config: VllmConfig, cut: int) -> bool:
|
||||
"""Whether this rank may replay from layer ``cut`` on; warns when not."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
spec_config = vllm_config.speculative_config
|
||||
draft_config = spec_config.draft_model_config if spec_config else None
|
||||
draft_hf_config = getattr(draft_config, "hf_config", None)
|
||||
draft_window = getattr(draft_hf_config, "sliding_window", None)
|
||||
draft_layer_types = getattr(draft_hf_config, "layer_types", None) or ()
|
||||
window = self.config.sliding_window
|
||||
if self.start_layer > cut or self.end_layer < self.config.num_hidden_layers:
|
||||
reason = (
|
||||
"the pipeline stage holding the last KV source layer must also "
|
||||
"hold every layer after it"
|
||||
)
|
||||
elif (
|
||||
self.use_sequence_parallel
|
||||
or parallel_config.prefill_context_parallel_size > 1
|
||||
or parallel_config.use_ubatching
|
||||
):
|
||||
reason = (
|
||||
"the replay-layer batch shrinks per rank, which sequence and "
|
||||
"prefill-context parallelism and microbatching cannot follow"
|
||||
)
|
||||
elif any(i >= cut for i in getattr(self.config, "engram_layer_ids", ())):
|
||||
reason = "an Engram layer sits at or after the last KV source layer"
|
||||
elif draft_config is not None and (
|
||||
draft_window is None
|
||||
or draft_window > window
|
||||
or any(t != "sliding_attention" for t in draft_layer_types)
|
||||
):
|
||||
reason = (
|
||||
f"the drafter (sliding window {draft_window}) reads hidden states "
|
||||
f"outside the target's {window}-token window"
|
||||
)
|
||||
else:
|
||||
return True
|
||||
logger.warning_once("Decoder SWA bounded replay is off: %s.", reason)
|
||||
return False
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
@@ -711,14 +984,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -799,14 +1064,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
@@ -1101,6 +1358,10 @@ class DeepseekV41LLMForCausalLM(
|
||||
engram_hash = self.model.engram_hash
|
||||
return engram_hash.lookback_depth if engram_hash is not None else 0
|
||||
|
||||
@property
|
||||
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
|
||||
return self.model.decoder_replay_layers
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# DeepSeek-V4.1-Flash mono decode layer (gfx950, FlyDSL)
|
||||
|
||||
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
|
||||
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
|
||||
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
|
||||
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
|
||||
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
|
||||
ATOM's V4.1 mono decode
|
||||
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
|
||||
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
|
||||
from it, each file naming its origin; the attention stages (`attention/`) are
|
||||
new, reading vLLM's KV records and weights as loaded.
|
||||
|
||||
## Scope
|
||||
|
||||
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
|
||||
settled, without folding the previous output in, on layer 0 (the embedding
|
||||
broadcast to the four streams) and after an Engram write (layers 1 and 14,
|
||||
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
|
||||
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
|
||||
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
|
||||
key table from that top-k. The FFN launch serves a layer outside these (an
|
||||
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
|
||||
The paths interleave freely (same tensors at the layer boundary).
|
||||
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
|
||||
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
|
||||
routing bias is `e_score_correction_bias`).
|
||||
- Contract: a drop-in for `DeepseekV4DecoderLayer.forward`:
|
||||
in `x` (the previous FFN output, reduced, bf16 [M, 5120]), `residual`
|
||||
(bf16 [M, 4, 5120]), `post_mix` [M, 4, 1], `res_mix` [M, 4, 4],
|
||||
`pre_mix` [M, 4] (f32, the previous seam's)
|
||||
out the MoE output (reduced), the residual after the FFN seam, and that
|
||||
seam's post / comb / pre mixes.
|
||||
|
||||
## Launches
|
||||
|
||||
K1 seam (stages/seam.py): slice (160 x 32 columns): fold x into the
|
||||
residual (post x + sum comb r), the collapsed input (pre . R'), partial
|
||||
mixes (fn . R') and sums of squares -> gate (S): 24 mixes -> pre / post
|
||||
/ comb (Sinkhorn 20) -> norm (S): attn_norm -> vLLM MXFP8 -> X8
|
||||
attention front (attention/front.py): wqkv, q / kv norms + KV insert,
|
||||
wq_b + RoPE -> Q
|
||||
K2 attention back (attention/back.py): split attention, combine + inverse
|
||||
RoPE, wo_a, wo_b; the bf16 wo_b partial pushed to every rank
|
||||
FFN seam: reduce (the ranks' partials summed in rank order, fp32 ->
|
||||
bf16: the attention output), slice, gate, norm -> the MoE input
|
||||
MoE (stages/moe.py): xq, router (bf16 GEMV, f32 logits), route
|
||||
(sqrtsoftplus + bias, top-6, renormalized x 1.5), shared expert (MXFP8
|
||||
GEMVs, clamped SiLU), ug (MXFP4 x MXFP8 per expert of the step's
|
||||
union), down (+ routing weight, top-k order; + shared) -> push, rank-order
|
||||
sum -> out
|
||||
FFN K2 without the attention back: push (160 x 32 columns) of vLLM's
|
||||
unreduced wo_b output, as its bits, to every rank's attention region,
|
||||
then K2's FFN seam and MoE (the same code, `layer.run_ffn`)
|
||||
|
||||
## Execution model and hand-offs
|
||||
|
||||
- Grid = 256 CTAs (one a CU, all resident), 512 threads a CTA, LDS a per-stage
|
||||
union. A stage is a task table: CTA `b` runs tasks `b, b + 256, ...`, stages
|
||||
in order; a consumer only waits on earlier stages, so no deadlock while the
|
||||
grid is resident. A GEMV task issues its weight loads before it polls its
|
||||
activations.
|
||||
- In a launch: tagged pairs (value + tag in one 8 B store) or plain data +
|
||||
flags, at device scope. The tag is the launch pair's epoch (K1 `2e + 1`,
|
||||
K2 or the FFN launch `2e + 2`), a device word their CTA 0 moves on once every
|
||||
CTA has marked the launch done: no clearing between steps, no host step hook, every argument
|
||||
a fixed device pointer (graph-safe).
|
||||
- Across ranks: one symmetric uncached buffer a rank (aiter `UncachedIpcHeap`),
|
||||
every rank holding every peer's address; pushes are system-scope stores of
|
||||
(value, tag) pairs. Every rank runs the same launches, so the epochs agree;
|
||||
peer regions alternate by epoch parity, and a rank cannot run two steps ahead
|
||||
of a peer (each step's reduce waits on every rank's push).
|
||||
|
||||
## Numerics (the vLLM path each stage reproduces)
|
||||
|
||||
- Seams: aiter `mhc_fused_post_pre_delayed_rmsnorm` (post fold order, fn as
|
||||
bf16 hi + lo, products summed in order, Triton sigmoid / exp / rcp).
|
||||
- Attention: vLLM's ROCm chain. `mxfp8(x)`: one E8M0 per 32 values,
|
||||
`code = clamp(ceil(log2(amax / 448)) + 127, 0, 254)`; weights in vLLM's
|
||||
row-major e4m3 [N, K] with [N/32, K/32] E8M0 scales (one shared copy); KV
|
||||
records as vLLM's insert writes them (dims 0..447 e4m3 with a UE8M0 per 64,
|
||||
RoPE dims bf16, 584 B); keys = the sliding window plus the top-k compressed
|
||||
rows, softmax in base 2 with the attention sink folded in.
|
||||
- MoE: vLLM's router (bf16 gate GEMV, f32 logits, `topk_hash_softplus_sqrt`)
|
||||
and AITER a8w4 `fused_moe` (interleaved gate / up, swiglu limit 10, weights
|
||||
in aiter's (16, 16) shuffle and `shuffle_scale` order); the shared expert as
|
||||
vLLM's MXFP8 block-32 linears with `silu_and_mul_with_clamp`.
|
||||
- All-reduces: exact, fp32 in rank order -> bf16. vLLM's decode all-reduces at
|
||||
M <= 48 are below its INT4 quick-reduce threshold, so vLLM runs them exact too.
|
||||
|
||||
## Validation
|
||||
|
||||
- Per layer against vLLM's own modules in a TP process group (`DeepseekV4MoE`,
|
||||
the mHC seams, the attention chain, the all-reduce), real checkpoint weights.
|
||||
- Bench: layers of one type back to back in one HIP graph, cold weights,
|
||||
us per layer, M = 1..48, TP2 / TP4; then e2e (AgentX, InferenceX#3690
|
||||
recipe).
|
||||
@@ -0,0 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1-Flash mono decode layer: one decoder layer of a decode step in
|
||||
two persistent FlyDSL launches, its TP all-reduces in-kernel (README.md). The
|
||||
host side is ``runner``; vLLM docks it in ``..mono_decode``."""
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The mono layer's attention stages: ``front`` (K1: wqkv, q / kv norms and the
|
||||
KV insert, wq_b) and ``back`` (K2: split attention, combine, wo_a, wo_b)."""
|
||||
@@ -0,0 +1,888 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K2 ``back``: one DeepSeek-V4.1 attention layer from the rotated query to the
|
||||
rank's wo_b partial.
|
||||
|
||||
One launch a layer and rank, ``BLOCKS`` x ``THREADS``, S rows:
|
||||
|
||||
split (S x chunks of CK keys): flash-decoding over one chunk of a token's
|
||||
keys (K1's key table: top-k compressed rows, then window rows) for
|
||||
every head of the rank. The token's q in LDS; each wave one
|
||||
(head tile, 16-key tile): its fp8_ds_mla rows loaded once,
|
||||
dequantized into its QK operands and kept raw in LDS; QK (bf16
|
||||
MFMA), the chunk's max / sum per head, bf16 p in LDS; PV over the
|
||||
chunk's raw rows (wave w: dims 64 w ..) -> the chunk's partial
|
||||
(bf16) and its (max, sum) per head, published
|
||||
combine (S x H): a (token, head)'s chunks merged with the sink folded in,
|
||||
bf16, inverse GPT-J RoPE of dims 448.., vLLM's MXFP8 of each 32
|
||||
-> XO (published)
|
||||
wo_a (G x 1024 / 32, two 16-row tiles): the group's XO rows times wo_a
|
||||
-> bf16 -> MXFP8 of the 32 rows -> X8B (published)
|
||||
wo_b (320, 16 rows): X8B times wo_b -> bf16 -> ``out``
|
||||
|
||||
Hand-offs are plain device-scope data behind tagged flags (half the bytes of a
|
||||
tagged pair per value). Adapted from ATOM's V4.1 ``attn_post`` (K2a), which
|
||||
reads a bf16 pool and ends in a peer all-reduce.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
from .device import (
|
||||
BLOCKS,
|
||||
CM_DEV,
|
||||
THREADS,
|
||||
WAVES,
|
||||
Mailbox,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
clamp_fp8,
|
||||
first_task,
|
||||
fp8_pack4,
|
||||
fp8x8_bf16,
|
||||
fresh,
|
||||
gload,
|
||||
gstore,
|
||||
hw_exp2,
|
||||
lane_gather,
|
||||
ld_f32,
|
||||
ld_i32,
|
||||
mfma_bf16,
|
||||
mx_code,
|
||||
mx_mul,
|
||||
pow2,
|
||||
rsrc,
|
||||
spin_until,
|
||||
stamp,
|
||||
traced,
|
||||
xshfl,
|
||||
)
|
||||
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
|
||||
from .plan import (
|
||||
DATA,
|
||||
HALF,
|
||||
HEAD_DIM,
|
||||
HEAD_TILE,
|
||||
KEYS,
|
||||
NOPE,
|
||||
ROPE,
|
||||
ROWS,
|
||||
WOA_ROWS,
|
||||
Dims,
|
||||
back_scratch,
|
||||
cdiv,
|
||||
front_end,
|
||||
tile_rows,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
_STREAM = fx.Stream(None)
|
||||
NEG = -3.4028234663852886e38
|
||||
LOG2E = 1.4426950408889634
|
||||
REC = DATA + 8 # a raw record row in LDS: 576 data bytes + 8 scale bytes
|
||||
Q_ROW = (
|
||||
HEAD_DIM // 2 + 4
|
||||
) # a q row in LDS (i32 words): 512 bf16 + 16 B, no bank aliasing
|
||||
P_PAD = 2 # i32 words past a P row: 16 head rows read together hit distinct banks
|
||||
# wo_a's K order: waves 0..3 / 4..7 take the two 16-row tiles, a contiguous K
|
||||
# range a wave, summed in wave order
|
||||
WOA_SPLIT_K = WAVES * ROWS // WOA_ROWS
|
||||
BACK_POINTS = 16 # start, stage ends, split internals, GEMV internals
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackBuild:
|
||||
tokens: int
|
||||
tp: int
|
||||
ratio: int # 0: window only; 1 / 2: top-k over a compressed cache
|
||||
timeline: bool = False
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
def head_groups(d: Dims, s: int) -> int:
|
||||
"""Wave groups a split unit's head tiles are spread over: two (a tile each,
|
||||
64-key chunks, twice the units) while the units fit the grid, else one (a
|
||||
wave's keys loaded and dequantized once for every head tile)."""
|
||||
return 2 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= BLOCKS else 1
|
||||
|
||||
|
||||
def chunk_keys(d: Dims, s: int) -> int:
|
||||
"""A split unit's keys: a 16-key tile a wave of a head group."""
|
||||
return 16 * (WAVES // head_groups(d, s))
|
||||
|
||||
|
||||
def p_row(d: Dims, s: int) -> int:
|
||||
"""A P row (one head's chunk of bf16 p) in LDS, i32 words."""
|
||||
return chunk_keys(d, s) // 2 + P_PAD
|
||||
|
||||
|
||||
@traced
|
||||
def publish(c, region, i):
|
||||
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if c["tid"] == 0:
|
||||
c["mb"].put(region, i, fx.Int32(1))
|
||||
|
||||
|
||||
@traced
|
||||
def await_flags(c, region, n, index):
|
||||
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
|
||||
tid = c["tid"]
|
||||
if tid < n:
|
||||
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- keys
|
||||
|
||||
|
||||
def key_rec(c, t, k):
|
||||
"""(data address, scale address, valid) of token t's key k from K1's key
|
||||
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
|
||||
none (it reads a zero record: its scores are masked, its values zeros)."""
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
# a table this launch wrote (``krdy``) is read past the CU's cache
|
||||
cm = CM_DEV if const_expr("krdy" in c) else 0
|
||||
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
|
||||
ok = val != fx.Int32(-1)
|
||||
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
|
||||
if const_expr(ratio > 0):
|
||||
is_c = ok & (val < 0)
|
||||
blk = is_c.select(a["comp_block"], a["swa_block"])
|
||||
stride = is_c.select(a["comp_stride"], a["swa_stride"])
|
||||
base = is_c.select(a["comp"], a["swa"])
|
||||
else:
|
||||
blk, stride, base = a["swa_block"], a["swa_stride"], a["swa"]
|
||||
b = slot // blk
|
||||
off = slot % blk
|
||||
page = base + fx.Int64(b) * stride
|
||||
z = a["zrec"]
|
||||
data = ok.select(page + fx.Int64(off * DATA), z)
|
||||
sc = ok.select(page + fx.Int64(blk * DATA + off * 8), z + fx.Int64(DATA))
|
||||
return data, sc, ok
|
||||
|
||||
|
||||
def _scale_byte(scw, blk64):
|
||||
return (scw[blk64 // 4] >> (blk64 % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def _fp8_pair_f32(word, scale, hi):
|
||||
v = rocdl.cvt_scalef32_pk_f32_fp8(
|
||||
fx.Vector.make_type(2, fx.Float32),
|
||||
as_ir_value(fx.Int32(word)),
|
||||
as_ir_value(fx.Float32(scale)),
|
||||
hi,
|
||||
)
|
||||
v = fx.Vector(v)
|
||||
return [v[0], v[1]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- split
|
||||
|
||||
|
||||
@traced
|
||||
def stage_split(c, unit):
|
||||
"""Flash-decoding over chunk ``unit % nchunk`` of token ``unit // nchunk``,
|
||||
every head of the rank -> the chunk's partial and (max, sum) per head. Wave
|
||||
(head group hg, key tile kt): keys 16 kt .. of the chunk (loaded and
|
||||
dequantized once) against the group's head tiles; then dims 64 w .. of the
|
||||
PV over every key and head."""
|
||||
tid, lane, wave, d, s = c["tid"], c["lane"], c["wave"], c["d"], c["S"]
|
||||
a, sl = c["args"], c["sl"]
|
||||
nchunk = c["nchunk"]
|
||||
t = unit // nchunk
|
||||
ch = unit % nchunk
|
||||
ck = chunk_keys(d, s)
|
||||
ngrp = head_groups(d, s)
|
||||
nkt = WAVES // ngrp # 16-key tiles a chunk
|
||||
nht = d.head_tiles // ngrp # head tiles a wave
|
||||
hgi = wave // nkt
|
||||
kt = wave % nkt
|
||||
g = lane // 16
|
||||
r16 = lane % 16
|
||||
k0 = ch * ck
|
||||
if const_expr("krdy" in c):
|
||||
await_flags(c, c["krdy"], 1, lambda u: t)
|
||||
# the key count, this lane's key and q: independent loads, out together
|
||||
# (the key table holds -1 past a token's keys)
|
||||
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
|
||||
key = k0 + kt * 16 + r16
|
||||
data, sc, okv = key_rec(c, t, key)
|
||||
qrow = d.heads * HEAD_DIM // 8
|
||||
qv = []
|
||||
for i in range_constexpr(cdiv(qrow, THREADS)):
|
||||
u = fresh(tid) + THREADS * i
|
||||
qv.append(
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(a["q"]),
|
||||
(t * d.q_rows) // 2 + fx.min(u, qrow - 1) * 4,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
)
|
||||
)
|
||||
if k0 < kv_len:
|
||||
# a key's 448 e4m3 as 7 x 16 B (lane g: dims 64 c + 16 g ..), its 64
|
||||
# bf16 rope dims as 2 x 16 B (32 c + 8 g ..), its 7 scale bytes
|
||||
scw = gload(sc, words=2)
|
||||
nope = [
|
||||
gload(data + fx.Int64(64 * c4 + 16 * g), words=4)
|
||||
for c4 in range(NOPE // 64)
|
||||
]
|
||||
rope = [
|
||||
gload(data + fx.Int64(NOPE + (cc * 32 + 8 * g) * 2), words=4)
|
||||
for cc in range(ROPE // 32)
|
||||
]
|
||||
# q of every head into LDS (bf16 rows of 512 + pad), 16 B a thread
|
||||
for i in range_constexpr(cdiv(qrow, THREADS)):
|
||||
u = fresh(tid) + THREADS * i
|
||||
if u < qrow:
|
||||
fx.ptr_store(
|
||||
qv[i],
|
||||
sl["q"]
|
||||
+ (u // (HEAD_DIM // 8)) * Q_ROW
|
||||
+ (u % (HEAD_DIM // 8)) * 4,
|
||||
)
|
||||
# the raw rows to LDS for the PV (head group 0's waves)
|
||||
if hgi == 0:
|
||||
rawb = sl["raw"] + (kt * 16 + r16) * (REC // 4)
|
||||
for c4 in range_constexpr(NOPE // 64):
|
||||
fx.ptr_store(nope[c4], rawb + (64 * c4 + 16 * g) // 4)
|
||||
for cc in range_constexpr(ROPE // 32):
|
||||
fx.ptr_store(rope[cc], rawb + (NOPE + (cc * 32 + 8 * g) * 2) // 4)
|
||||
if g == 0:
|
||||
fx.ptr_store(scw, rawb + DATA // 4)
|
||||
gpu.barrier()
|
||||
stamp(c, 5)
|
||||
# QK: A = the keys (dequantized once), B = q of a head tile (LDS); the
|
||||
# contraction's 32-dim chunk 2 c + h of the nope dims is {64 c + 16 g +
|
||||
# 8 h ..} (lane g's 16 B load), q read in the same order
|
||||
scale = fx.Int32(a["qk_scale"]).bitcast(fx.Float32)
|
||||
st = sl["st"]
|
||||
accs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(nht)]
|
||||
for c4 in range_constexpr(NOPE // 64):
|
||||
ksc = pow2(_scale_byte(scw, c4))
|
||||
for h2 in range_constexpr(2):
|
||||
kw = fp8x8_bf16(nope[c4][2 * h2], nope[c4][2 * h2 + 1], ksc).bitcast(
|
||||
fx.BFloat16
|
||||
)
|
||||
for ht in range_constexpr(nht):
|
||||
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
|
||||
qw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
qb + 32 * c4 + 8 * g + 4 * h2,
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
|
||||
for cc in range_constexpr(ROPE // 32):
|
||||
kw = rope[cc].bitcast(fx.BFloat16)
|
||||
for ht in range_constexpr(nht):
|
||||
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
|
||||
qw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
qb + (NOPE + cc * 32 + 8 * g) // 2,
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
|
||||
# scores of keys 4 g + i (C rows), head r16 of a tile (C column)
|
||||
ok = (key < kv_len) & okv
|
||||
okw = ok.select(fx.Int32(1), fx.Int32(0))
|
||||
valid = [lane_gather(okw, 4 * g + i) != 0 for i in range(4)]
|
||||
svs = []
|
||||
for ht in range_constexpr(nht):
|
||||
sv = [
|
||||
fx.Float32(valid[i].select(accs[ht][i] * scale, fx.Float32(NEG)))
|
||||
for i in range(4)
|
||||
]
|
||||
mw = butterfly(
|
||||
fx.max(fx.max(sv[0], sv[1]), fx.max(sv[2], sv[3])), (32, 16), fx.max
|
||||
)
|
||||
if g == 0:
|
||||
fx.ptr_store(mw, st + ((hgi * nht + ht) * nkt + kt) * HEAD_TILE + r16)
|
||||
svs.append(sv)
|
||||
gpu.barrier()
|
||||
lst = st + d.heads * nkt
|
||||
for ht in range_constexpr(nht):
|
||||
hti = hgi * nht + ht
|
||||
m = fx.ptr_load(st + (hti * nkt) * HEAD_TILE + r16)
|
||||
for j in range_constexpr(1, nkt):
|
||||
m = fx.max(m, fx.ptr_load(st + (hti * nkt + j) * HEAD_TILE + r16))
|
||||
sv = svs[ht]
|
||||
p = [
|
||||
fx.Float32(valid[i].select(hw_exp2(sv[i] - m), fx.Float32(0.0)))
|
||||
for i in range(4)
|
||||
]
|
||||
lw = (p[0] + p[2]) + (p[1] + p[3])
|
||||
lw = lw + xshfl(lw, 32)
|
||||
lw = lw + xshfl(lw, 16)
|
||||
pw = (
|
||||
fx.Vector.from_elements(p, fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)
|
||||
)
|
||||
pbase = (
|
||||
sl["p"] + (hti * HEAD_TILE + r16) * p_row(d, s) + (kt * 16 + 4 * g) // 2
|
||||
)
|
||||
fx.ptr_store(fx.Vector.from_elements([pw[0], pw[1]], fx.Int32), pbase)
|
||||
if g == 0:
|
||||
fx.ptr_store(lw, lst + (hti * nkt + kt) * HEAD_TILE + r16)
|
||||
gpu.barrier()
|
||||
# chunk stats: (max, sum) per head
|
||||
if tid < d.heads:
|
||||
hts = tid // HEAD_TILE
|
||||
hr = tid % HEAD_TILE
|
||||
mm = fx.ptr_load(st + (hts * nkt) * HEAD_TILE + hr)
|
||||
ll = fx.ptr_load(lst + (hts * nkt) * HEAD_TILE + hr)
|
||||
for j in range_constexpr(1, nkt):
|
||||
mm = fx.max(mm, fx.ptr_load(st + (hts * nkt + j) * HEAD_TILE + hr))
|
||||
ll = ll + fx.ptr_load(lst + (hts * nkt + j) * HEAD_TILE + hr)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([mm, ll], fx.Float32),
|
||||
rsrc(c["pstat"]),
|
||||
((t * nchunk + ch) * d.heads + tid) * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
# PV: wave w dims 64 w + 4 r16 + dt (dt = 0..3: one LDS word holds the
|
||||
# four of a key); the rope wave (dims 448..) reads bf16 pairs. Lane
|
||||
# (r16, g) takes keys 4 g + j of a 16-key tile: A[dim][key]
|
||||
is_rope = wave == WAVES - 1
|
||||
dbase = 64 * wave
|
||||
sh = (wave % 4) * 8
|
||||
accs = [
|
||||
[fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(4)]
|
||||
for _ in range(d.head_tiles)
|
||||
]
|
||||
for kt2 in range_constexpr(nkt):
|
||||
rb0 = sl["raw"] + (kt2 * 16 + 4 * g) * (REC // 4)
|
||||
vv = fx.Vector.filled(16, 0.0, fx.Float32) # [dt][j]
|
||||
if is_rope:
|
||||
rv = []
|
||||
for j in range_constexpr(4):
|
||||
rw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
rb0 + j * (REC // 4) + (NOPE + 8 * r16) // 4,
|
||||
result_type=fx.Vector.make_type(2, fx.Int32),
|
||||
)
|
||||
)
|
||||
rv.append([bf_lo(rw[0]), bf_hi(rw[0]), bf_lo(rw[1]), bf_hi(rw[1])])
|
||||
vv = fx.Vector.from_elements(
|
||||
[rv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
|
||||
)
|
||||
else:
|
||||
nv = []
|
||||
for j in range_constexpr(4):
|
||||
rb = rb0 + j * (REC // 4)
|
||||
w = fx.Int32(fx.ptr_load(rb + (dbase + 4 * r16) // 4))
|
||||
scb = fx.Int32(fx.ptr_load(rb + DATA // 4 + wave // 4))
|
||||
scl = pow2((scb >> sh) & 0xFF)
|
||||
nv.append(
|
||||
_fp8_pair_f32(w, scl, False) + _fp8_pair_f32(w, scl, True)
|
||||
)
|
||||
vv = fx.Vector.from_elements(
|
||||
[nv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
|
||||
)
|
||||
for ht2 in range_constexpr(d.head_tiles):
|
||||
pb = fx.Vector(
|
||||
fx.ptr_load(
|
||||
sl["p"]
|
||||
+ (ht2 * HEAD_TILE + r16) * p_row(d, s)
|
||||
+ (kt2 * 16 + 4 * g) // 2,
|
||||
result_type=fx.Vector.make_type(2, fx.Int32),
|
||||
)
|
||||
).bitcast(fx.Int16)
|
||||
for dt in range_constexpr(4):
|
||||
av = (
|
||||
fx.Vector.from_elements(
|
||||
[vv[dt * 4 + j] for j in range(4)], fx.Float32
|
||||
)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int16)
|
||||
)
|
||||
accs[ht2][dt] = fx.Vector(
|
||||
rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.vec(4, T.f32), [av, pb, accs[ht2][dt]]
|
||||
)
|
||||
)
|
||||
stamp(c, 6)
|
||||
# partial[t][ch][head][dim] bf16: lane (r16, g) holds C rows 4 g + i ->
|
||||
# dims 64 w + 16 g + 4 i + dt of head ht2 * 16 + r16: 32 contiguous
|
||||
# bytes, two 16 B stores (4 lanes fill a 128 B line)
|
||||
for ht2 in range_constexpr(d.head_tiles):
|
||||
hh = ht2 * HEAD_TILE + r16
|
||||
words = []
|
||||
for i in range_constexpr(4):
|
||||
v4 = fx.Vector.from_elements(
|
||||
[accs[ht2][dt][i] for dt in range(4)], fx.Float32
|
||||
)
|
||||
w2 = v4.to(fx.BFloat16).bitcast(fx.Int32)
|
||||
words += [w2[0], w2[1]]
|
||||
off = (((t * nchunk + ch) * d.heads + hh) * HEAD_DIM + dbase + 16 * g) // 2
|
||||
for hlf in range_constexpr(2):
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words[4 * hlf : 4 * hlf + 4], fx.Int32),
|
||||
rsrc(c["part"]),
|
||||
off + 4 * hlf,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
publish(c, c["srdy"], t * nchunk + ch)
|
||||
stamp(c, 7)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- combine
|
||||
|
||||
COMB_HEADS = WAVES # heads a combine unit: a wave each
|
||||
|
||||
|
||||
@traced
|
||||
def stage_combine(c, unit):
|
||||
"""(token, 8 heads) ``unit``, a wave a head, 8 dims a lane: the chunks
|
||||
merged with the sink folded in, bf16, inverse RoPE, MXFP8 of each 32 -> XO,
|
||||
published (a flag a (token, head group))."""
|
||||
lane, wave, d, s = c["lane"], c["wave"], c["d"], c["S"]
|
||||
a = c["args"]
|
||||
hg_n = d.heads // COMB_HEADS
|
||||
t = unit // hg_n
|
||||
hg = unit % hg_n
|
||||
h = hg * COMB_HEADS + wave
|
||||
nchunk = c["nchunk"]
|
||||
ck = chunk_keys(d, s)
|
||||
kv_len = ld_i32(a["klen"], t)
|
||||
nlive = (kv_len + ck - 1) // ck
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
sink = ld_f32(a["sink"], h) * fx.Float32(LOG2E)
|
||||
d0 = lane * 8 # this lane's 8 dims
|
||||
rope_lane = d0 >= NOPE
|
||||
jj = fx.max((d0 - NOPE) // 2, 0) # 4 pairs from here
|
||||
cs = [
|
||||
rope_lane.select(ld_f32(a["cos_sin"], pos * ROPE + jj + i), fx.Float32(1.0))
|
||||
for i in range(4)
|
||||
]
|
||||
sn = [
|
||||
rope_lane.select(
|
||||
ld_f32(a["cos_sin"], pos * ROPE + HALF + jj + i), fx.Float32(0.0)
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
await_flags(c, c["srdy"], nlive, lambda u: t * nchunk + u)
|
||||
m = fx.Float32(NEG)
|
||||
ms, ls, vs = [], [], []
|
||||
for cc in range_constexpr(nchunk):
|
||||
live = cc < nlive
|
||||
stv = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(c["pstat"]),
|
||||
((t * nchunk + cc) * d.heads + h) * 2,
|
||||
vec_width=2,
|
||||
dtype=T.f32,
|
||||
)
|
||||
)
|
||||
pv = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(c["part"]),
|
||||
(((t * nchunk + cc) * d.heads + h) * HEAD_DIM + d0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
)
|
||||
mc = fx.Float32(live.select(stv[0], fx.Float32(NEG)))
|
||||
ms.append(mc)
|
||||
ls.append(fx.Float32(live.select(stv[1], fx.Float32(0.0))))
|
||||
vals = []
|
||||
for e in range_constexpr(4):
|
||||
vals += [bf_lo(pv[e]), bf_hi(pv[e])]
|
||||
vs.append([fx.Float32(live.select(v, fx.Float32(0.0))) for v in vals])
|
||||
m = fx.max(m, mc)
|
||||
m_final = fx.max(m, sink)
|
||||
lsum = hw_exp2(sink - m_final)
|
||||
tot = [fx.Float32(0.0)] * 8
|
||||
for cc in range_constexpr(nchunk):
|
||||
wgt = hw_exp2(ms[cc] - m_final)
|
||||
lsum = lsum + ls[cc] * wgt
|
||||
tot = [tot[e] + vs[cc][e] * wgt for e in range(8)]
|
||||
inv = fx.Float32(1.0) / lsum
|
||||
o = [bf16_round(tot[e] * inv) for e in range(8)]
|
||||
# inverse GPT-J RoPE of each pair: e' = e cos + o sin, o' = o cos - e sin
|
||||
y = []
|
||||
for i in range_constexpr(4):
|
||||
e, od = o[2 * i], o[2 * i + 1]
|
||||
y += [e * cs[i] + od * sn[i], od * cs[i] - e * sn[i]]
|
||||
amax = abs(y[0])
|
||||
for v in y[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max) # 32 dims = 4 lanes
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(y[i] * mul) for i in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(y[4 + i] * mul) for i in range(4)])
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["xo"]),
|
||||
t * (d.q_rows // 4) + (h * HEAD_DIM + d0) // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["xos"]),
|
||||
t * (d.q_rows // 32) + (h * HEAD_DIM + d0) // 32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
publish(c, c["crdy"], t * hg_n + hg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wo_a / wo_b
|
||||
|
||||
|
||||
@traced
|
||||
def stage_woa(c, task, ops, t0, n):
|
||||
"""Rows 32 task .. of wo_a -> bf16 -> vLLM's MXFP8 of the 32 -> X8B."""
|
||||
tid, red, d, lane = c["tid"], c["red"], c["d"], c["lane"]
|
||||
split = WOA_SPLIT_K
|
||||
gemv_mfmas(c, d.group_k, c["xl"], c["xsl"], red, ops, split, tiled=True, rows=n)
|
||||
gpu.barrier()
|
||||
mine = tid < WOA_ROWS * n
|
||||
tl = fx.min(tid // WOA_ROWS, n - 1)
|
||||
t = plus(t0, tl)
|
||||
row = tid % WOA_ROWS
|
||||
tile = row // ROWS
|
||||
y = bf16_round(
|
||||
row_sum(red, row % ROWS, tl, [tile * split + w for w in range(split)])
|
||||
)
|
||||
amax = butterfly(abs(y), (1, 2, 4, 8, 16), fx.max)
|
||||
code = mx_code(amax)
|
||||
q = clamp_fp8(y * mx_mul(code))
|
||||
w = fp8_pack4(
|
||||
q,
|
||||
*[
|
||||
lane_gather(q.bitcast(fx.Int32), fx.min(lane + k, 63)).bitcast(fx.Float32)
|
||||
for k in (1, 2, 3)
|
||||
],
|
||||
)
|
||||
if mine & (row % 4 == 0):
|
||||
bo.buffer_store(
|
||||
w,
|
||||
rsrc(c["x8b"]),
|
||||
t * (d.o_rows // 4) + (task * WOA_ROWS + row) // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if mine & (row == 0):
|
||||
bo.buffer_store(
|
||||
code, rsrc(c["x8bs"]), t * (d.o_rows // 32) + task, cache_modifier=CM_DEV
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
def woa_loads(c, task):
|
||||
a = c["args"]
|
||||
return gemv_loads(
|
||||
c,
|
||||
a["woa"],
|
||||
a["woa_s"],
|
||||
c["d"].group_k,
|
||||
task * WOA_ROWS,
|
||||
WOA_SPLIT_K,
|
||||
tiled=True,
|
||||
)
|
||||
|
||||
|
||||
def wob_loads(c, task, pred=None):
|
||||
a = c["args"]
|
||||
return gemv_loads(c, a["wob"], a["wob_s"], c["d"].o_rows, task * ROWS, pred=pred)
|
||||
|
||||
|
||||
def _wob_store(a, t, col, v0, v1):
|
||||
word = (
|
||||
fx.Vector.from_elements([v0, v1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
bo.buffer_store(word, rsrc(a["out"]), (t * a["out_stride"] + col) // 2)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wob(c, task, ops, task2, ops2, has2, t0, n, mark=False):
|
||||
"""Rows 16 task .. (and 16 task2 .., ``has2``) of wo_b -> bf16 -> out, or
|
||||
(``c["wob_out"]``, the mono layer) the caller's hook with each bf16 pair."""
|
||||
tid, red = c["tid"], c["red"]
|
||||
o_rows = c["d"].o_rows
|
||||
a = c["args"]
|
||||
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red, ops, rows=n)
|
||||
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red + RED, ops2, rows=n)
|
||||
gpu.barrier()
|
||||
if const_expr(mark):
|
||||
stamp(c, 15)
|
||||
per = ROWS * n // 2 # bf16 pairs a task
|
||||
k = tid // per
|
||||
emit = c.get("wob_out") or (lambda t, col, v0, v1: _wob_store(a, t, col, v0, v1))
|
||||
if (tid < 2 * per) & ((k == 0) | has2):
|
||||
u = tid % per
|
||||
rp = u % (ROWS // 2)
|
||||
tl = u // (ROWS // 2)
|
||||
t = plus(t0, tl)
|
||||
rk = red + k * RED
|
||||
col = (k == 0).select(task, task2) * ROWS + 2 * rp
|
||||
v0 = row_sum(rk, 2 * rp, tl)
|
||||
v1 = row_sum(rk, 2 * rp + 1, tl)
|
||||
emit(t, col, v0, v1)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kernel
|
||||
|
||||
|
||||
def back_plan(key):
|
||||
"""(nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob) of a build."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
ck = chunk_keys(d, s)
|
||||
nchunk = cdiv(KEYS, ck)
|
||||
n_split = s * nchunk
|
||||
n_comb = s * (d.heads // COMB_HEADS)
|
||||
COMB0 = n_split % BLOCKS
|
||||
assert d.woa_tasks <= BLOCKS
|
||||
assert BLOCKS < d.wob_tasks <= 2 * BLOCKS
|
||||
# the GEMVs past the attention units, so CTAs without attention work have
|
||||
# their weights in flight from the start
|
||||
WOA0 = (n_split + n_comb) % BLOCKS
|
||||
WOB0 = (WOA0 + d.woa_tasks) % BLOCKS
|
||||
# wo_b's weights before the wo_a stage only while the wo_a CTAs have no
|
||||
# attention work (loads complete in issue order: else they would hold up
|
||||
# wo_a's activation rows)
|
||||
early_wob = n_split + n_comb + d.woa_tasks <= BLOCKS
|
||||
return nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob
|
||||
|
||||
|
||||
def back_lds_members(s: int, d: Dims) -> dict:
|
||||
"""K2's attention LDS structs: the split stage's and the GEMVs' (a union's
|
||||
members: one at a time)."""
|
||||
rows = tile_rows(s)
|
||||
ck = chunk_keys(d, s)
|
||||
|
||||
@fx.struct
|
||||
class SplitLds:
|
||||
q: fx.Array[fx.Int32, d.heads * Q_ROW, 16]
|
||||
raw: fx.Array[fx.Int32, ck * REC // 4, 16]
|
||||
p: fx.Array[fx.Int32, d.heads * p_row(d, s), 16]
|
||||
st: fx.Array[fx.Float32, 2 * d.heads * (WAVES // head_groups(d, s)), 16]
|
||||
|
||||
@fx.struct
|
||||
class GemvLds:
|
||||
red: fx.Array[fx.Float32, 2 * RED, 16]
|
||||
xl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 4), 16]
|
||||
xsl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 32), 16]
|
||||
|
||||
return {"split": SplitLds, "gemv": GemvLds}
|
||||
|
||||
|
||||
def back_lds(s: int, d: Dims):
|
||||
"""K2's attention LDS: the union of ``back_lds_members``."""
|
||||
m = back_lds_members(s, d)
|
||||
SplitLds, GemvLds = m["split"], m["gemv"]
|
||||
|
||||
@fx.union
|
||||
class BackLds:
|
||||
split: SplitLds # type: ignore[valid-type]
|
||||
gemv: GemvLds # type: ignore[valid-type]
|
||||
|
||||
return BackLds
|
||||
|
||||
|
||||
def back_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
|
||||
"""The attention back's context: LDS views (``lds``: an allocated
|
||||
``back_lds``, or a dict of the members' views), mailbox (tag ``tag``),
|
||||
scratch regions, arguments by name."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
tid = fx.thread_idx.x
|
||||
if isinstance(lds, dict):
|
||||
spl, gml = lds["split"], lds["gemv"]
|
||||
else:
|
||||
spl, gml = lds.split.peek(), lds.gemv.peek()
|
||||
c = {
|
||||
"S": s,
|
||||
"tid": tid,
|
||||
"bid": fx.block_idx.x,
|
||||
"lane": tid % 64,
|
||||
"wave": tid // 64,
|
||||
"red": gml.red.ptr,
|
||||
"xl": gml.xl.ptr,
|
||||
"xsl": gml.xsl.ptr,
|
||||
"d": d,
|
||||
"ratio": key.ratio,
|
||||
"mb": Mailbox(tag),
|
||||
"nchunk": back_plan(key)[0],
|
||||
"sl": {"q": spl.q.ptr, "raw": spl.raw.ptr, "p": spl.p.ptr, "st": spl.st.ptr},
|
||||
"args": args,
|
||||
}
|
||||
for region, (off, _) in (
|
||||
layout or back_scratch(s, d, start=front_end(s, d))
|
||||
).items():
|
||||
c[region] = scratch + fx.Int64(off)
|
||||
return c
|
||||
|
||||
|
||||
EPOCH_MARKS = 4 # word offset of the per-CTA marks in the epoch buffer
|
||||
|
||||
|
||||
@traced
|
||||
def epoch_begin(c, epoch, ep):
|
||||
"""This CTA has read the launch pair's epoch: its own mark word (no
|
||||
contended atomic) := ep + 1, which ``epoch_end`` checks."""
|
||||
if c["tid"] == 0:
|
||||
bo.buffer_store(
|
||||
ep + 1, rsrc(epoch), EPOCH_MARKS + c["bid"], cache_modifier=CM_DEV
|
||||
)
|
||||
|
||||
|
||||
@traced
|
||||
def epoch_end(c, epoch, ep, reset=None):
|
||||
"""CTA 0, once every CTA marked this epoch (thread u checks CTA u's mark):
|
||||
``reset()`` (state later pairs must find cleared), then the epoch moved on.
|
||||
The next pair's hand-offs carry a tag no earlier pair wrote; later launches'
|
||||
kernels start after this one ends."""
|
||||
if c["bid"] == 0:
|
||||
tid = c["tid"]
|
||||
if tid < BLOCKS:
|
||||
r = rsrc(epoch)
|
||||
|
||||
def load_mark():
|
||||
return fx.Int32(
|
||||
bo.buffer_load(
|
||||
r,
|
||||
EPOCH_MARKS + tid,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
)
|
||||
|
||||
_ = spin_until(load_mark, lambda v: v != ep + 1)
|
||||
gpu.barrier()
|
||||
if tid == 0:
|
||||
if const_expr(reset is not None):
|
||||
reset()
|
||||
gstore(epoch, ep + 1, words=1)
|
||||
|
||||
|
||||
@traced
|
||||
def run_back(c, key, bid):
|
||||
"""The attention back's stages on this CTA: split, combine, wo_a, wo_b."""
|
||||
s = key.tokens
|
||||
dd = Dims(key.tp)
|
||||
# plain ints: a Python object read inside traced control flow is carried
|
||||
woa_tasks, groups, heads, group_k = dd.woa_tasks, dd.groups, dd.heads, dd.group_k
|
||||
q_rows, o_rows, wob_tasks = dd.q_rows, dd.o_rows, dd.wob_tasks
|
||||
a = c["args"]
|
||||
tiles = token_tiles(s)
|
||||
_, n_split, n_comb, COMB0, WOA0, WOB0, early_wob = back_plan(key)
|
||||
stamp(c, 0)
|
||||
for unit in range(first_task(bid, 0), n_split, BLOCKS):
|
||||
stage_split(c, unit)
|
||||
stamp(c, 1)
|
||||
for unit in range(first_task(bid, COMB0), n_comb, BLOCKS):
|
||||
stage_combine(c, unit)
|
||||
gpu.barrier()
|
||||
stamp(c, 2)
|
||||
# the GEMV weights, every CTA right after its attention work: its wo_a
|
||||
# task (if any), then its one or two wo_b tasks
|
||||
woa_task = first_task(bid, WOA0)
|
||||
has_woa = woa_task < woa_tasks
|
||||
woa_t = fx.min(woa_task, woa_tasks - 1)
|
||||
ops = gemv_loads(
|
||||
c,
|
||||
a["woa"],
|
||||
a["woa_s"],
|
||||
group_k,
|
||||
woa_t * WOA_ROWS,
|
||||
WOA_SPLIT_K,
|
||||
tiled=True,
|
||||
pred=has_woa,
|
||||
)
|
||||
wob_task = first_task(bid, WOB0)
|
||||
wob_second = wob_task + BLOCKS
|
||||
has_second = wob_second < wob_tasks
|
||||
|
||||
def wob_weights():
|
||||
return (
|
||||
wob_loads(c, wob_task),
|
||||
wob_loads(c, fx.min(wob_second, wob_tasks - 1), pred=has_second),
|
||||
)
|
||||
|
||||
if const_expr(early_wob):
|
||||
ops1, ops2 = wob_weights()
|
||||
if const_expr(c.get("tl") is not None):
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
stamp(c, 8)
|
||||
# wo_a: every token tile of its group's XO rows, the next tile's in flight
|
||||
if has_woa:
|
||||
grp = woa_t // (woa_tasks // groups)
|
||||
await_flags(c, c["crdy"], s, lambda u: u * (heads // COMB_HEADS) + grp)
|
||||
stamp(c, 9)
|
||||
xo = (
|
||||
c["xo"],
|
||||
c["xos"],
|
||||
q_rows // 4,
|
||||
grp * group_k // 4,
|
||||
group_k // 4,
|
||||
q_rows // 32,
|
||||
grp * group_k // 32,
|
||||
group_k // 32,
|
||||
)
|
||||
regs = rows_loads(c, *xo, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, group_k // 4, group_k // 32, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *xo, *tiles[ti + 1])
|
||||
stage_woa(c, woa_t, ops, t0, n)
|
||||
stamp(c, 10)
|
||||
publish(c, c["ardy"], woa_t)
|
||||
stamp(c, 3)
|
||||
# wo_b
|
||||
if const_expr(not early_wob):
|
||||
ops1, ops2 = wob_weights()
|
||||
stamp(c, 11)
|
||||
await_flags(c, c["ardy"], woa_tasks, lambda u: u)
|
||||
stamp(c, 12)
|
||||
x8b = (
|
||||
c["x8b"],
|
||||
c["x8bs"],
|
||||
o_rows // 4,
|
||||
0,
|
||||
o_rows // 4,
|
||||
o_rows // 32,
|
||||
0,
|
||||
o_rows // 32,
|
||||
)
|
||||
regs = rows_loads(c, *x8b, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, o_rows // 4, o_rows // 32, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti == 0):
|
||||
stamp(c, 14)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *x8b, *tiles[ti + 1])
|
||||
stage_wob(
|
||||
c,
|
||||
wob_task,
|
||||
ops1,
|
||||
fx.min(wob_second, wob_tasks - 1),
|
||||
ops2,
|
||||
has_second,
|
||||
t0,
|
||||
n,
|
||||
mark=ti == 0,
|
||||
)
|
||||
if const_expr(ti == 0):
|
||||
stamp(c, 13)
|
||||
stamp(c, 4)
|
||||
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Device primitives of the DeepSeek-V4.1 attention mega kernel.
|
||||
|
||||
Wave reductions and shuffles, hardware math, bf16 / fp8 / MX packing, the
|
||||
scaled MFMA, buffer and 64-bit global accesses, and the tagged-pair mailbox.
|
||||
Adapted from ATOM's mono decode framework (``atom/mono/device``), itself ported
|
||||
from the FlyDSL GLM-5 layer kernel.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
|
||||
from aiter.ops.flydsl.kernels.kernels_common import kernel_signature
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import llvm
|
||||
from flydsl.compiler.ast_rewriter import ASTRewriter
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
# The execution model: one CTA a CU of an MI355X, all resident.
|
||||
BLOCKS = 256
|
||||
THREADS = 512
|
||||
WAVES = THREADS // 64
|
||||
|
||||
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16, NT = 2. SC1 alone is
|
||||
# device scope (past the per-XCD L2s); NT streams data read once.
|
||||
CM_DEV = 16
|
||||
CM_NT = 2
|
||||
|
||||
FP8_MAX = 448.0
|
||||
FP8, FP4 = 0, 4 # f8f6f4 MFMA operand formats
|
||||
UNIT_SCALE = 127 # E8M0 code of 1.0
|
||||
FLT_MIN = 1.1754943508222875e-38
|
||||
|
||||
POLL_MAX = 12 # mailbox pairs polled a batch
|
||||
|
||||
|
||||
def traced(fn):
|
||||
"""FlyDSL's AST rewrite for a helper, so its ``if`` / ``while`` / ``for`` on
|
||||
traced values lower to scf ops (kernels rewrite only their own body)."""
|
||||
return ASTRewriter.transform(fn)
|
||||
|
||||
|
||||
def kernel_symbol(stem, **params):
|
||||
"""Kernel symbol that profilers demangle to ``aiter::<stem>_<params>``."""
|
||||
name = f"{stem}_{kernel_signature(**params)}"
|
||||
return f"_ZN5aiter{len(name)}{name}E"
|
||||
|
||||
|
||||
def first_task(bid, base):
|
||||
"""CTA ``bid``'s first task of a stage placed from CTA ``base``, then every
|
||||
BLOCKS-th."""
|
||||
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- memory
|
||||
|
||||
|
||||
def rsrc(addr, nbytes=None):
|
||||
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
|
||||
|
||||
|
||||
def ld_i32(ptr, i, cm=0):
|
||||
return fx.Int32(
|
||||
bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.i32, cache_modifier=cm)
|
||||
)
|
||||
|
||||
|
||||
def ld_f32(ptr, i):
|
||||
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
|
||||
|
||||
|
||||
def ld_bf(ptr, i):
|
||||
return fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
)
|
||||
|
||||
|
||||
def ld_i64(ptr, i):
|
||||
"""Element i of an i64 buffer (two dwords)."""
|
||||
w = fx.Vector(bo.buffer_load(rsrc(ptr), i * 2, vec_width=2, dtype=T.i32))
|
||||
return (fx.Int64(w[1]) << 32) | fx.Int64(fx.Uint32(w[0]))
|
||||
|
||||
|
||||
def _gptr(addr):
|
||||
return llvm.IntToPtrOp(
|
||||
ir.Type.parse("!llvm.ptr<1>"), as_ir_value(fx.Int64(addr))
|
||||
).result
|
||||
|
||||
|
||||
def gload(addr, words=4, nt=False):
|
||||
"""``words`` i32 (1, 2 or 4) at a 64-bit global address: a cache past 4 GiB is
|
||||
reachable only this way (a buffer offset is 32 bits)."""
|
||||
i32 = ir.IntegerType.get_signless(32)
|
||||
ty = i32 if words == 1 else ir.VectorType.get([words], i32)
|
||||
v = llvm.LoadOp(ty, _gptr(addr), alignment=4 * words, nontemporal=nt or None).result
|
||||
return fx.Int32(v) if words == 1 else fx.Vector(v)
|
||||
|
||||
|
||||
def atomic_add_dev(addr, value):
|
||||
"""Device-scope atomic add on the global i32 at ``addr``; the old value.
|
||||
Relaxed: an acquire / release here would write back and invalidate L2."""
|
||||
return fx.Int32(
|
||||
fx.atomic_add(
|
||||
_gptr(addr),
|
||||
fx.Int32(value),
|
||||
syncscope="agent",
|
||||
ordering=fx.AtomicOrdering.Monotonic,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def gstore(addr, value, words=4):
|
||||
"""Store ``words`` i32 (an Int32 or a Vector) at a 64-bit global address."""
|
||||
v = as_ir_value(fx.Int32(value) if words == 1 else value)
|
||||
llvm.StoreOp(v, _gptr(addr), alignment=4 * words)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- math
|
||||
|
||||
|
||||
def fresh(v):
|
||||
"""``v`` through an opaque move, so the lane math built on it is computed at
|
||||
its use instead of being hoisted out of a task loop and held in registers."""
|
||||
return fx.Int32(
|
||||
llvm.inline_asm(
|
||||
T.i32,
|
||||
[fx.Int32(v).ir_value()],
|
||||
"v_mov_b32 $0, $1",
|
||||
"=v,v",
|
||||
has_side_effects=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def tile_map(tid, width):
|
||||
"""(tid // width, tid % width) for 0 <= tid < THREADS without a division:
|
||||
shifts for a power of two, else a multiply-shift exact on that range."""
|
||||
if width & (width - 1) == 0:
|
||||
sh = width.bit_length() - 1
|
||||
return tid >> sh, tid & (width - 1)
|
||||
m = -(-(1 << 20) // width)
|
||||
assert all((t * m) >> 20 == t // width for t in range(THREADS)), width
|
||||
row = (tid * m) >> 20
|
||||
return row, tid - row * width
|
||||
|
||||
|
||||
def uniform(v):
|
||||
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
|
||||
|
||||
|
||||
def _hw_f32(name, x):
|
||||
return fx.Float32(
|
||||
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
|
||||
)
|
||||
|
||||
|
||||
def hw_rsq(x):
|
||||
return _hw_f32("llvm.amdgcn.rsq.f32", x)
|
||||
|
||||
|
||||
def hw_rcp(x):
|
||||
return _hw_f32("llvm.amdgcn.rcp.f32", x)
|
||||
|
||||
|
||||
def hw_exp2(x):
|
||||
return _hw_f32("llvm.amdgcn.exp2.f32", x)
|
||||
|
||||
|
||||
def fma(a, b, c):
|
||||
return fx.Float32(fmath.fma(fx.Float32(a), fx.Float32(b), fx.Float32(c)))
|
||||
|
||||
|
||||
def div_rn(x, d, r):
|
||||
"""``x / d`` rounded to nearest (IEEE), given ``r = RN(1 / d)``."""
|
||||
q = x * r
|
||||
return fma(fma(-q, d, x), r, q)
|
||||
|
||||
|
||||
def permlane_swap(off, x, y):
|
||||
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
|
||||
pr = swap(
|
||||
llvm.StructType.get_literal([T.i32, T.i32]),
|
||||
as_ir_value(fx.Int32(x)),
|
||||
as_ir_value(fx.Int32(y)),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
|
||||
|
||||
|
||||
def lane_gather(v, src_lane):
|
||||
"""Lane ``src_lane``'s value of v (ds_bpermute)."""
|
||||
return fx.Int32(
|
||||
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
|
||||
)
|
||||
|
||||
|
||||
def lane_f32(v, src_lane):
|
||||
return lane_gather(v.bitcast(fx.Int32), src_lane).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def readlane(v, src_lane):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
r = fx.Int32(rocdl.readlane(T.i32, bits, fx.Int32(src_lane)))
|
||||
return r.bitcast(fx.Float32) if isinstance(v, fx.Float32) else r
|
||||
|
||||
|
||||
def xshfl(v, off):
|
||||
"""Value of lane ``lane ^ off`` (DPP in a row, permlane swaps across)."""
|
||||
assert off in (1, 2, 4, 8, 16, 32), off
|
||||
if off >= 16:
|
||||
return v.shuffle_xor(off, 64)
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
if off == 8:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
|
||||
elif off == 4:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
|
||||
elif off == 2:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
|
||||
else:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
|
||||
return y.bitcast(fx.Float32) if is_f else y
|
||||
|
||||
|
||||
def xred(v, off, op):
|
||||
if off < 16:
|
||||
return op(v, xshfl(v, off))
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
a, b = permlane_swap(off, x, x)
|
||||
if is_f:
|
||||
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
|
||||
return op(type(v)(a), type(v)(b))
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return a + b
|
||||
|
||||
|
||||
def butterfly(v, offsets, op=_add):
|
||||
"""Butterfly reduction over the xor ``offsets``, in that order."""
|
||||
for off in offsets:
|
||||
v = xred(v, off, op)
|
||||
return v
|
||||
|
||||
|
||||
def wave_sum(v):
|
||||
return butterfly(v, (32, 16, 8, 4, 2, 1))
|
||||
|
||||
|
||||
def wave_max(v):
|
||||
return butterfly(v, (32, 16, 8, 4, 2, 1), fx.max)
|
||||
|
||||
|
||||
def row_shr(v, lane, k):
|
||||
"""Lane - k of the same 16-lane row, else 0 (DPP row_shr)."""
|
||||
moved = lane_gather(v.bitcast(fx.Int32), fx.max(lane - k, 0)).bitcast(fx.Float32)
|
||||
return fx.Float32((lane % 16 >= k).select(moved, fx.Float32(0.0)))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- packing
|
||||
|
||||
|
||||
def bf16_round(a):
|
||||
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
|
||||
|
||||
|
||||
def bf16_pair(a, b):
|
||||
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
|
||||
return (
|
||||
fx.Vector.from_elements([a, b], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Float32)[0]
|
||||
)
|
||||
|
||||
|
||||
def bf_lo(w):
|
||||
return (w << 16).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf_hi(w):
|
||||
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def fp8_pack4(a, b, c, d):
|
||||
"""Four f32 (in range) -> one dword of four E4M3 bytes, RNE."""
|
||||
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
|
||||
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
|
||||
|
||||
|
||||
def fp8x8_bf16(w0, w1, scale):
|
||||
"""Eight E4M3 of two dwords times an f32 ``scale`` -> eight bf16 as a
|
||||
4-dword Vector (exact for a power-of-two scale in range)."""
|
||||
out = []
|
||||
for w in (w0, w1):
|
||||
for hi in (False, True):
|
||||
p = rocdl.cvt_scalef32_pk_bf16_fp8(
|
||||
T.bf16x2, as_ir_value(fx.Int32(w)), as_ir_value(fx.Float32(scale)), hi
|
||||
)
|
||||
out.append(fx.Vector(p).bitcast(fx.Int32)[0])
|
||||
return fx.Vector.from_elements(out, fx.Int32)
|
||||
|
||||
|
||||
def clamp_fp8(v):
|
||||
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
||||
|
||||
|
||||
def pow2(code):
|
||||
"""The fp32 power of two a biased exponent (1..254) stands for."""
|
||||
return (code << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def rcp_pow2(code):
|
||||
"""1 / ``pow2(code)``, exact for code in [1, 253]."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def ceil_exp(v):
|
||||
"""Biased exponent of the smallest power of two >= v (v > 0 finite, normal)."""
|
||||
bits = v.bitcast(fx.Int32)
|
||||
return ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
||||
fx.Int32(1), fx.Int32(0)
|
||||
)
|
||||
|
||||
|
||||
def mx_code(amax):
|
||||
"""VLLM's ROCm MXFP8 scale code of a 32-group: clamp(ceil(log2(amax / 448))
|
||||
+ 127, 0, 254), amax floored at FLT_MIN."""
|
||||
a = fx.max(amax, fx.Float32(FLT_MIN))
|
||||
q = div_rn(a, fx.Float32(FP8_MAX), fx.Float32(1.0 / FP8_MAX))
|
||||
return fx.min(fx.max(ceil_exp(q), fx.Int32(0)), fx.Int32(254))
|
||||
|
||||
|
||||
def mx_mul(code):
|
||||
"""2^(127 - code) as fp32, for code in [0, 254] (the quant multiplier)."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- MFMA
|
||||
|
||||
|
||||
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8):
|
||||
"""16x16x128 scaled MFMA (gfx950 f8f6f4): lane l (row l % 16) holds K bytes
|
||||
16 (l // 16) .. +16 and 64 + 16 (l // 16) .. +16 of the 128-step, and the
|
||||
E8M0 of K block l // 16."""
|
||||
return fx.Vector(
|
||||
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
||||
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, 0, sa, 0, sb]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def mfma_bf16(a, b, c):
|
||||
"""16x16x32 bf16 MFMA: lane l (row / col l % 16) holds K 8 (l // 16) .. +8."""
|
||||
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- mailbox
|
||||
|
||||
|
||||
@traced
|
||||
def _spin(load_all, pending):
|
||||
v = load_all()
|
||||
while pending(v):
|
||||
rocdl.s_sleep(1)
|
||||
v = load_all()
|
||||
return v
|
||||
|
||||
|
||||
def spin_until(load, pending):
|
||||
"""``load()`` again (s_sleep between) while ``pending(value)``; the value."""
|
||||
return _spin(load, pending)
|
||||
|
||||
|
||||
class Mailbox:
|
||||
"""Tagged-pair hand-off between the CTAs of one launch (device scope).
|
||||
|
||||
Every 32-bit value is stored next to the launch's tag in one 8-byte store; a
|
||||
consumer polls until every tag matches, so a hand-off is one round trip with
|
||||
no separate flag. The tag is ``epoch * 64 + layer + 1`` (``epoch`` a device
|
||||
counter bumped once a step): a pair from an earlier layer or step never
|
||||
carries a live tag, and nothing is cleared between launches."""
|
||||
|
||||
def __init__(self, tag):
|
||||
self.tag = tag
|
||||
|
||||
def put(self, base, i, v):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
||||
rsrc(base),
|
||||
i * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def put_bf(self, base, i, vs):
|
||||
"""Elements i .. i + len(vs) (2 or 4, i aligned) as bf16 pairs, one store."""
|
||||
words = []
|
||||
for j in range_constexpr(len(vs) // 2):
|
||||
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(base),
|
||||
i,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def put_words(self, base, i, words):
|
||||
"""Raw 32-bit words (1 or 2) at pairs i.. (fp8 words must come this way)."""
|
||||
vals = []
|
||||
for w in words:
|
||||
vals += [fx.Int32(w), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(vals, fx.Int32),
|
||||
rsrc(base),
|
||||
i * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def poll(self, specs, batch=POLL_MAX):
|
||||
"""Batched poll of ``specs`` = [(base, pair index, npairs in {1, 2})]: all
|
||||
re-loaded while any tag is stale. Returns the value words."""
|
||||
if const_expr(len(specs) == 0):
|
||||
return []
|
||||
if const_expr(len(specs) > batch):
|
||||
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
||||
tag = self.tag
|
||||
|
||||
def load_all():
|
||||
words = []
|
||||
for b, i, n in specs:
|
||||
w = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(b),
|
||||
fx.Int32(i) * 2,
|
||||
vec_width=2 * n,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
)
|
||||
words += [w[e] for e in range(2 * n)]
|
||||
return fx.Vector.from_elements(words, fx.Int32)
|
||||
|
||||
nw = sum(2 * n for _, _, n in specs)
|
||||
|
||||
def pending(v):
|
||||
bad = v[1] != tag
|
||||
for e in range_constexpr(3, nw, 2):
|
||||
bad = bad | (v[e] != tag)
|
||||
return bad
|
||||
|
||||
v = _spin(load_all, pending)
|
||||
outs, e = [], 0
|
||||
for _, _, n in specs:
|
||||
outs.append([v[e + 2 * q] for q in range(n)])
|
||||
e += 2 * n
|
||||
return outs
|
||||
|
||||
|
||||
LOAD_BATCH = 4
|
||||
|
||||
|
||||
@traced
|
||||
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
|
||||
"""A CTA's strided loop over v = tid + THREADS i < n: ``batch`` rounds' loads
|
||||
issued before their uses; a round past n loads element n - 1, uses nothing."""
|
||||
rounds = -(-n // THREADS)
|
||||
for b0 in range_constexpr(0, rounds, batch):
|
||||
idx = range(b0, min(b0 + batch, rounds))
|
||||
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
|
||||
for i, g in zip(idx, got):
|
||||
v = tid + THREADS * i
|
||||
if v < n:
|
||||
use(v, g)
|
||||
|
||||
|
||||
def barrier():
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- timeline
|
||||
|
||||
|
||||
def memrealtime():
|
||||
"""s_memrealtime: the 100 MHz device clock."""
|
||||
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
|
||||
|
||||
|
||||
@traced
|
||||
def stamp(c, point):
|
||||
"""A timeline build's stamp: thread 0 of the CTA stores the clock at
|
||||
``point`` (``c["tl"]`` [BLOCKS, points] i64; None: not a timeline build)."""
|
||||
if const_expr(c.get("tl") is None):
|
||||
return
|
||||
if c["tid"] == 0:
|
||||
t = memrealtime()
|
||||
w = fx.Vector.from_elements([fx.Int32(t), fx.Int32(t >> 32)], fx.Int32)
|
||||
bo.buffer_store(w, rsrc(c["tl"]), (c["bid"] * c["tl_points"] + point) * 2)
|
||||
@@ -0,0 +1,680 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K1 ``front``: one DeepSeek-V4.1 attention layer from its normed input row to
|
||||
the rotated query and the step's KV rows in the sliding-window cache.
|
||||
|
||||
One launch a layer and rank, ``BLOCKS`` CTAs x ``THREADS`` threads, S rows.
|
||||
Every CTA issues its weight loads (its wqkv part and its wq_b tasks) at the
|
||||
kernel start, before it waits on anything; the stages:
|
||||
|
||||
xq (S > 16: S x 2 K halves, on the 32 CTAs without a wqkv part): the
|
||||
bf16 input's half -> vLLM's MXFP8 -> X8 (published, a flag each)
|
||||
kt (S): the token's attention keys for K2 -- top-k compressed slots
|
||||
through the kv source's block table, then window slots -> KT
|
||||
wqkv_a (112 row tiles x 2 K halves, 16 rows): the input's MXFP8 half --
|
||||
quantized in the CTA (S <= 16) or X8's -- a token tile at a time
|
||||
(the next tile's rows in flight) times wqkv's -> fp32 partials ->
|
||||
QKVP
|
||||
qkv (S): the halves summed -> bf16; q_norm -> bf16 -> MXFP8 -> QX8
|
||||
(published, a flag a token); kv_norm -> bf16 -> GPT-J RoPE -> bf16
|
||||
-> the fp8_ds_mla record at the token's slot
|
||||
wq_b (H x 512 / 16, 16 rows): QX8 times wq_b -> bf16 -> GPT-J RoPE of
|
||||
each head's dims 448.. -> q (global, bf16)
|
||||
|
||||
Each elementwise step follows vLLM's ROCm path (``mxfp8_quantize``,
|
||||
``FusedQKVRMSNorm``, ``fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert``);
|
||||
the GEMV reductions differ from the Triton GEMMs' only in their order.
|
||||
Adapted from ATOM's V4.1 ``attn_pre`` (K1), less its mHC seam.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from .device import (
|
||||
BLOCKS,
|
||||
CM_DEV,
|
||||
THREADS,
|
||||
Mailbox,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
ceil_exp,
|
||||
clamp_fp8,
|
||||
div_rn,
|
||||
first_task,
|
||||
fp8_pack4,
|
||||
fresh,
|
||||
gstore,
|
||||
hw_rsq,
|
||||
ld_f32,
|
||||
ld_i32,
|
||||
mx_code,
|
||||
mx_mul,
|
||||
rsrc,
|
||||
stamp,
|
||||
tile_map,
|
||||
traced,
|
||||
xshfl,
|
||||
)
|
||||
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
|
||||
from .plan import (
|
||||
DATA,
|
||||
EPS,
|
||||
HALF,
|
||||
HEAD_DIM,
|
||||
HIDDEN,
|
||||
KEYS,
|
||||
KV_DIM,
|
||||
NOPE,
|
||||
Q_RANK,
|
||||
QKV_ROWS,
|
||||
ROPE,
|
||||
ROWS,
|
||||
TILE,
|
||||
TOPK,
|
||||
WINDOW,
|
||||
WQKV_KPARTS,
|
||||
WQKV_TASKS,
|
||||
Dims,
|
||||
front_scratch,
|
||||
tile_rows,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
_STREAM = fx.Stream(None)
|
||||
X_WORDS = HIDDEN // 4
|
||||
X_GROUPS = HIDDEN // 32
|
||||
QX_WORDS = Q_RANK // 4
|
||||
KSTEPS = HIDDEN // 128 // WQKV_KPARTS # a wqkv part's 128-steps
|
||||
KWORDS = KSTEPS * 32
|
||||
KGROUPS = KSTEPS * 4
|
||||
XH = HIDDEN // WQKV_KPARTS # a K half's values
|
||||
XH_COLS = XH // 8 // 2 # 16 B items (8 values) a half of a K half
|
||||
XH_RPR = THREADS // XH_COLS # half-rows a round
|
||||
XQ_ITEMS = XH // 8 # an xq unit's 16 B items (one a thread)
|
||||
FRONT_POINTS = 6 # start, kt done, wqkv done, qkv done, wq_b done, (spare)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FrontBuild:
|
||||
tokens: int
|
||||
tp: int
|
||||
ratio: int # the layer's compress ratio (0: no top-k keys)
|
||||
timeline: bool = False
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
@traced
|
||||
def publish(c, region, i):
|
||||
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if c["tid"] == 0:
|
||||
c["mb"].put(region, i, fx.Int32(1))
|
||||
|
||||
|
||||
@traced
|
||||
def await_flags(c, region, n, index):
|
||||
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
|
||||
tid = c["tid"]
|
||||
if tid < n:
|
||||
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- xq
|
||||
|
||||
|
||||
def xq_loads(c, unit, pred):
|
||||
"""(token, K half) ``unit``'s bf16 input: its load only, 8 values a thread
|
||||
(threads past the half mirror its last item)."""
|
||||
a = c["args"]
|
||||
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
|
||||
col = fx.min(c["tid"], XQ_ITEMS - 1)
|
||||
e = (unit >> 1) * a["x_stride"] + (unit & 1) * XH + col * 8
|
||||
return fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_xq(c, unit, w, live):
|
||||
"""``xq_loads``' values -> vLLM's MXFP8 -> X8 words / X8S codes of the
|
||||
unit (published by the caller)."""
|
||||
tid, lane = c["tid"], c["lane"]
|
||||
vals = []
|
||||
for d in range_constexpr(4):
|
||||
vals += [bf_lo(w[d]), bf_hi(w[d])]
|
||||
amax = abs(vals[0])
|
||||
for v in vals[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max)
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
|
||||
t = unit >> 1
|
||||
part = unit & 1
|
||||
if live & (tid < XQ_ITEMS):
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["x8"]),
|
||||
t * X_WORDS + part * KWORDS + tid * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["x8s"]),
|
||||
t * X_GROUPS + part * KGROUPS + (tid >> 2),
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kt
|
||||
|
||||
|
||||
@traced
|
||||
def stage_kt(c, t):
|
||||
"""Token t's keys for K2: ``KT[t][k] = slot | 1 << 31`` for its top-k compressed
|
||||
rows (k < ntopk), the window slot for k in [ntopk, kv_len), -1 for an
|
||||
invalid or absent key; KLEN[t] = kv_len (0 for a pad row), KLEN[S + t] =
|
||||
ntopk."""
|
||||
tid = c["tid"]
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
# device scope where this launch reads the table (K2's)
|
||||
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
|
||||
slot = ld_i32(a["slot"], 2 * t)
|
||||
live = slot >= 0
|
||||
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
|
||||
if const_expr(ratio > 0):
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
ntopk = live.select(fx.min((pos + 1) // ratio, fx.Int32(TOPK)), fx.Int32(0))
|
||||
req = ld_i32(a["t2r"], t)
|
||||
else:
|
||||
ntopk = fx.Int32(0)
|
||||
kv_len = ntopk + nswa
|
||||
for i in range_constexpr((KEYS + THREADS - 1) // THREADS):
|
||||
k = fresh(tid) + THREADS * i
|
||||
sidx = fx.min(fx.max(k - ntopk, 0), WINDOW - 1)
|
||||
s_slot = ld_i32(a["swa_idx"], t * WINDOW + sidx)
|
||||
val = (k < kv_len).select(s_slot, fx.Int32(-1))
|
||||
if const_expr(ratio > 0):
|
||||
local = ld_i32(a["topk"], t * TOPK + fx.min(k, TOPK - 1))
|
||||
lc = fx.max(local, 0)
|
||||
e = a["comp_block"]
|
||||
blk = ld_i32(a["comp_bt"], req * a["bt_stride"] + lc // e)
|
||||
c_slot = (blk * e + lc % e) | fx.Int32(-2147483648)
|
||||
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
|
||||
val = (k < ntopk).select(c_val, val)
|
||||
if k < KEYS:
|
||||
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
|
||||
if tid == 0:
|
||||
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
|
||||
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wqkv_a
|
||||
|
||||
|
||||
def x_half_loads(c, part, pred, t0, n):
|
||||
"""Tokens t0 .. t0 + n's bf16 input, K half ``part``: its loads only (16 B a
|
||||
lane-item, 8 values; a thread one column of [2 n half-rows, 160]), masked
|
||||
off on CTAs without a wqkv part."""
|
||||
a = c["args"]
|
||||
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
|
||||
hr0, col = tile_map(c["tid"], XH_COLS)
|
||||
e0 = part * XH + col * 8
|
||||
out = []
|
||||
for i in range((2 * n + XH_RPR - 1) // XH_RPR):
|
||||
hr = fx.min(hr0 + XH_RPR * i, 2 * n - 1)
|
||||
e = (t0 + (hr >> 1)) * a["x_stride"] + (hr & 1) * (XH // 2) + e0
|
||||
out.append(fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32)))
|
||||
return out
|
||||
|
||||
|
||||
@traced
|
||||
def quant_x_half(c, regs, n):
|
||||
"""``x_half_loads``' values -> vLLM's MXFP8 rows of the K half in LDS (a
|
||||
32-group = 4 lanes)."""
|
||||
tid, lane = c["tid"], c["lane"]
|
||||
x_row, s_row = lds_row(KWORDS), lds_row(KGROUPS)
|
||||
hr0, col = tile_map(tid, XH_COLS)
|
||||
act = tid < XH_RPR * XH_COLS
|
||||
for i in range_constexpr(len(regs)):
|
||||
hr = hr0 + XH_RPR * i
|
||||
w = regs[i]
|
||||
vals = []
|
||||
for d in range_constexpr(4):
|
||||
vals += [bf_lo(w[d]), bf_hi(w[d])]
|
||||
amax = abs(vals[0])
|
||||
for v in vals[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max)
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
|
||||
tok = hr >> 1
|
||||
half = hr & 1
|
||||
if act & (hr < 2 * n):
|
||||
fx.ptr_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
c["xl"] + (tok * x_row + half * (KWORDS // 2) + col * 2),
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
fx.ptr_store(
|
||||
code, c["xsl"] + (tok * s_row + half * (KGROUPS // 2) + (col >> 2))
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wqkv(c, task, part, ops, t0, n, live):
|
||||
"""Rows 16 task .. of wqkv over K half ``part`` against tokens t0 .. t0 + n
|
||||
-> fp32 partials QKVP[part] (``live``: a CTA with a wqkv part)."""
|
||||
tid, red = c["tid"], c["red"]
|
||||
gemv_mfmas(c, HIDDEN, c["xl"], c["xsl"], red, ops, rows=n, nsteps=KSTEPS)
|
||||
gpu.barrier()
|
||||
if (tid < ROWS * n) & live:
|
||||
r = tid % ROWS
|
||||
tl = tid // ROWS
|
||||
t = plus(t0, tl)
|
||||
v = row_sum(red, r, tl)
|
||||
c["mb"].put(c["qkvp"], (part * c["S"] + t) * QKV_ROWS + task * ROWS + r, v)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- q / kv norms
|
||||
|
||||
|
||||
@traced
|
||||
def stage_qkv(c, t):
|
||||
"""Token t: the K halves summed -> bf16; q_norm -> bf16 -> MXFP8 (QX8);
|
||||
kv_norm -> bf16 -> RoPE -> bf16 -> the fp8_ds_mla record at
|
||||
slot_mapping[t]. 256 threads (8 q columns and 2 kv columns each); threads
|
||||
256.. mirror thread 255 and store nothing."""
|
||||
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
|
||||
a, s = c["args"], c["S"]
|
||||
tt = fx.min(tid, 255)
|
||||
mine = tid < 256
|
||||
qc = [fx.min(tt * 8 + j, Q_RANK - 1) for j in range(8)]
|
||||
# independent loads first: they overlap the partials' wait
|
||||
qw = [c["ldbf"](a["qn_w"], qc[j]) for j in range(8)]
|
||||
kw = [c["ldbf"](a["kvn_w"], tt * 2 + j) for j in range(2)]
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
slot = ld_i32(a["slot"], 2 * t)
|
||||
jr = tt - NOPE // 2
|
||||
is_rope = jr >= 0
|
||||
jj = fx.max(jr, 0)
|
||||
cs = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + jj), fx.Float32(1.0))
|
||||
sn = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + HALF + jj), fx.Float32(0.0))
|
||||
specs = []
|
||||
for p in range_constexpr(2):
|
||||
specs += [(c["qkvp"], (p * s + t) * QKV_ROWS + qc[j], 1) for j in range(8)]
|
||||
specs += [
|
||||
(c["qkvp"], (p * s + t) * QKV_ROWS + Q_RANK + tt * 2 + j, 1)
|
||||
for j in range(2)
|
||||
]
|
||||
got = c["mb"].poll(specs)
|
||||
vals = [
|
||||
bf16_round(got[j][0].bitcast(fx.Float32) + got[10 + j][0].bitcast(fx.Float32))
|
||||
for j in range(10)
|
||||
]
|
||||
xq = [((tt * 8 + j) < Q_RANK).select(vals[j], fx.Float32(0.0)) for j in range(8)]
|
||||
xk = vals[8:10]
|
||||
ssq = xq[0] * xq[0]
|
||||
for j in range_constexpr(1, 8):
|
||||
ssq = ssq + xq[j] * xq[j]
|
||||
ssq = butterfly(ssq, (32, 16, 8, 4, 2, 1))
|
||||
ssk = butterfly(xk[0] * xk[0] + xk[1] * xk[1], (32, 16, 8, 4, 2, 1))
|
||||
if (lane == 0) & mine:
|
||||
fx.ptr_store(ssq, red + wave)
|
||||
fx.ptr_store(ssk, red + 4 + wave)
|
||||
gpu.barrier()
|
||||
tq = (fx.ptr_load(red + 0) + fx.ptr_load(red + 1)) + (
|
||||
fx.ptr_load(red + 2) + fx.ptr_load(red + 3)
|
||||
)
|
||||
tk = (fx.ptr_load(red + 4) + fx.ptr_load(red + 5)) + (
|
||||
fx.ptr_load(red + 6) + fx.ptr_load(red + 7)
|
||||
)
|
||||
rq = hw_rsq(
|
||||
div_rn(tq, fx.Float32(float(Q_RANK)), fx.Float32(1.0 / Q_RANK))
|
||||
+ fx.Float32(EPS)
|
||||
)
|
||||
rk = hw_rsq(
|
||||
div_rn(tk, fx.Float32(float(KV_DIM)), fx.Float32(1.0 / KV_DIM))
|
||||
+ fx.Float32(EPS)
|
||||
)
|
||||
# q: bf16 normed row, then vLLM's MXFP8 (a 32-group = 4 threads)
|
||||
ys = [bf16_round((xq[j] * rq) * qw[j]) for j in range(8)]
|
||||
amax = abs(ys[0])
|
||||
for y in ys[1:]:
|
||||
amax = fx.max(amax, abs(y))
|
||||
code = mx_code(butterfly(amax, (1, 2), fx.max))
|
||||
mul = mx_mul(code)
|
||||
if const_expr(c.get("index", False)): # noqa: SIM102
|
||||
# the bf16 qr, vLLM's indexer's input
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
words = [
|
||||
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
for i in range(4)
|
||||
]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(a["qr_out"]),
|
||||
t * (Q_RANK // 2) + 4 * tt,
|
||||
)
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["qx8"]),
|
||||
t * QX_WORDS + 2 * tt,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if tt % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["qx8s"]),
|
||||
t * (Q_RANK // 32) + tt // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
# kv: this thread's pair (2 tt, 2 tt + 1)
|
||||
kn = [bf16_round((xk[j] * rk) * kw[j]) for j in range(2)]
|
||||
e0 = bf16_round(kn[0] * cs - kn[1] * sn)
|
||||
e1 = bf16_round(kn[0] * sn + kn[1] * cs)
|
||||
# record quant of the nope dims: a 64-block = 32 threads (half a wave)
|
||||
amax = butterfly(fx.max(abs(e0), abs(e1)), (1, 2, 4, 8, 16), fx.max)
|
||||
rcode = ceil_exp(
|
||||
div_rn(
|
||||
fx.max(amax, fx.Float32(1e-4)), fx.Float32(448.0), fx.Float32(1.0 / 448.0)
|
||||
)
|
||||
)
|
||||
inv = mx_mul(rcode)
|
||||
q0 = clamp_fp8(e0 * inv)
|
||||
q1 = clamp_fp8(e1 * inv)
|
||||
n0 = xshfl(q0, 1)
|
||||
n1 = xshfl(q1, 1)
|
||||
word = fp8_pack4(q0, q1, n0, n1)
|
||||
rope_word = (
|
||||
fx.Vector.from_elements([e0, e1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
blk = a["swa_block"]
|
||||
sblock = slot // blk
|
||||
soff = slot % blk
|
||||
base = a["swa"] + fx.Int64(sblock) * fx.Int64(a["swa_stride"])
|
||||
rec = base + fx.Int64(soff * DATA)
|
||||
if mine & (tt % 32 == 0) & (tt < NOPE // 2):
|
||||
fx.ptr_store(rcode.bitcast(fx.Float32), red + 8 + tt // 32)
|
||||
gpu.barrier()
|
||||
if mine & (slot >= 0):
|
||||
if tt < NOPE // 2:
|
||||
if tt % 2 == 0:
|
||||
gstore(rec + fx.Int64(2 * tt), word, words=1)
|
||||
else:
|
||||
gstore(rec + fx.Int64(NOPE + 4 * (tt - NOPE // 2)), rope_word, words=1)
|
||||
if tt == 0:
|
||||
sb = [fx.ptr_load(red + 8 + b).bitcast(fx.Int32) for b in range(7)]
|
||||
w0 = sb[0] | (sb[1] << 8) | (sb[2] << 16) | (sb[3] << 24)
|
||||
w1 = sb[4] | (sb[5] << 8) | (sb[6] << 16)
|
||||
sc = base + fx.Int64(blk * DATA + soff * 8)
|
||||
gstore(sc, fx.Vector.from_elements([w0, w1], fx.Int32), words=2)
|
||||
publish(c, c["qrdy"], t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wq_b
|
||||
|
||||
|
||||
def wqb_rope(c, task0, pos_t, n):
|
||||
"""(cos, sin) of this thread's pair in ``stage_wqb``'s epilogue over a tile
|
||||
of n rows (``pos_t``: its token's position; every task of the CTA shares
|
||||
the pair's dims: tasks BLOCKS apart are whole heads apart)."""
|
||||
a = c["args"]
|
||||
rp = c["tid"] % (ROWS // 2)
|
||||
dim = (task0 * ROWS) % HEAD_DIM + 2 * rp
|
||||
jj = fx.max((dim - NOPE) >> 1, 0)
|
||||
return ld_f32(a["cos_sin"], pos_t * ROPE + jj), ld_f32(
|
||||
a["cos_sin"], pos_t * ROPE + HALF + jj
|
||||
)
|
||||
|
||||
|
||||
def wqb_pos(c, t0, n):
|
||||
"""This thread's token position for ``stage_wqb``'s epilogue of tile
|
||||
t0 .. t0 + n."""
|
||||
per = ROWS * n // 2
|
||||
tl = fx.min((c["tid"] % per) // (ROWS // 2), n - 1)
|
||||
return ld_i32(c["args"]["pos"], 2 * (t0 + tl))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wqb(c, task0, bops, t0, n, rope):
|
||||
"""Rows 16 (task0 + BLOCKS k) .. of the rank's H x 512 query rows, every
|
||||
task k of the CTA: GEMVs -> bf16 -> GPT-J RoPE of a head's dims 448..
|
||||
(fp32) -> bf16 -> q."""
|
||||
tid, red, d = c["tid"], c["red"], c["d"]
|
||||
a = c["args"]
|
||||
nt = len(bops)
|
||||
for k in range_constexpr(nt):
|
||||
gemv_mfmas(c, Q_RANK, c["qxl"], c["qxsl"], red + k * RED, bops[k], rows=n)
|
||||
gpu.barrier()
|
||||
per = ROWS * n // 2 # bf16 pairs a task
|
||||
if tid < nt * per:
|
||||
k = tid // per
|
||||
u = tid % per
|
||||
rp = u % (ROWS // 2)
|
||||
tl = u // (ROWS // 2)
|
||||
t = plus(t0, tl)
|
||||
r0 = (task0 + BLOCKS * k) * ROWS + 2 * rp
|
||||
rk = red + k * RED
|
||||
v0 = bf16_round(row_sum(rk, 2 * rp, tl))
|
||||
v1 = bf16_round(row_sum(rk, 2 * rp + 1, tl))
|
||||
is_rope = (task0 * ROWS) % HEAD_DIM + 2 * rp >= NOPE
|
||||
cs, sn = rope
|
||||
e0 = is_rope.select(v0 * cs - v1 * sn, v0)
|
||||
e1 = is_rope.select(v0 * sn + v1 * cs, v1)
|
||||
word = (
|
||||
fx.Vector.from_elements([e0, e1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
bo.buffer_store(word, rsrc(a["q"]), (t * d.q_rows + r0) // 2)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kernel
|
||||
|
||||
|
||||
def front_lds(s: int, d: Dims):
|
||||
"""K1's LDS (the attention front's stages)."""
|
||||
rows = tile_rows(s)
|
||||
wqb_per_cta = d.wqb_tasks // BLOCKS
|
||||
|
||||
@fx.struct
|
||||
class FrontLds:
|
||||
red: fx.Array[fx.Float32, wqb_per_cta * RED + 16, 16]
|
||||
qxl: fx.Array[fx.Int32, rows * lds_row(QX_WORDS), 16]
|
||||
qxsl: fx.Array[fx.Int32, rows * lds_row(Q_RANK // 32), 16]
|
||||
xl: fx.Array[fx.Int32, rows * lds_row(KWORDS), 16]
|
||||
xsl: fx.Array[fx.Int32, rows * lds_row(KGROUPS), 16]
|
||||
|
||||
return FrontLds
|
||||
|
||||
|
||||
def front_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
|
||||
"""The front stages' context: LDS views, mailbox (tag ``tag``), scratch
|
||||
regions, arguments (``args``: the kernel's, by name)."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
tid = fx.thread_idx.x
|
||||
c = {
|
||||
"S": s,
|
||||
"tid": tid,
|
||||
"bid": fx.block_idx.x,
|
||||
"lane": tid % 64,
|
||||
"wave": tid // 64,
|
||||
"red": lds.red.ptr,
|
||||
"xl": lds.xl.ptr,
|
||||
"xsl": lds.xsl.ptr,
|
||||
"qxl": lds.qxl.ptr,
|
||||
"qxsl": lds.qxsl.ptr,
|
||||
"mb": Mailbox(tag),
|
||||
"d": d,
|
||||
"ratio": key.ratio,
|
||||
"ldbf": lambda ptr, i: fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
),
|
||||
"args": args,
|
||||
}
|
||||
for region, (off, _) in (layout or front_scratch(s, d)).items():
|
||||
c[region] = scratch + fx.Int64(off)
|
||||
return c
|
||||
|
||||
|
||||
@traced
|
||||
def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
|
||||
"""The front's stages on this CTA. ``x8_given``: a stage before this one
|
||||
(the mono layer's seam norm) publishes X8 / XRDY for every token (no input
|
||||
quantization here). ``before_wqkv()`` runs ahead of the wqkv weights' loads
|
||||
(a latency-critical stage of the caller's), ``after_wqkv()`` right after."""
|
||||
s = key.tokens
|
||||
wqb_tasks = Dims(key.tp).wqb_tasks
|
||||
a = c["args"]
|
||||
tid = c["tid"]
|
||||
tiles = token_tiles(s)
|
||||
wqb_per_cta = wqb_tasks // BLOCKS
|
||||
assert wqb_tasks % BLOCKS == 0
|
||||
n_wqkv = WQKV_TASKS * WQKV_KPARTS # 224
|
||||
assert n_wqkv <= BLOCKS
|
||||
n_free = BLOCKS - n_wqkv # CTAs without a wqkv part: xq, kt
|
||||
QKV0 = 0 # on wqkv CTAs
|
||||
WQB0 = 0
|
||||
in_cta_quant = s <= TILE and not x8_given
|
||||
xq_stage = s > TILE and not x8_given
|
||||
xq_units = WQKV_KPARTS * s
|
||||
xq_per = (xq_units + n_free - 1) // n_free
|
||||
stamp(c, 0)
|
||||
|
||||
# ---- Loads complete in issue order, so every CTA issues its latency-
|
||||
# critical loads before its later stages' weights: a wqkv CTA its first
|
||||
# input tile and its wqkv part, the next tiles while it quantizes, then
|
||||
# its wq_b tasks; a CTA with a key-table task those after that work.
|
||||
# The q / kv norms run on wqkv CTAs, their partials' wait behind the
|
||||
# wq_b weights that are due by then anyway.
|
||||
wqkv_unit = first_task(bid, 0)
|
||||
has_wqkv = wqkv_unit < n_wqkv
|
||||
task = fx.min(wqkv_unit, n_wqkv - 1) % WQKV_TASKS
|
||||
part = fx.min(wqkv_unit, n_wqkv - 1) // WQKV_TASKS
|
||||
wqb_task = first_task(bid, WQB0)
|
||||
free = bid - n_wqkv
|
||||
is_free = free >= 0
|
||||
if const_expr(in_cta_quant):
|
||||
xregs = [x_half_loads(c, part, has_wqkv, *tiles[0])]
|
||||
if const_expr(xq_stage):
|
||||
xunits = [free + n_free * i for i in range(xq_per)]
|
||||
xlive = [is_free & (u < xq_units) for u in xunits]
|
||||
xq_regs = [
|
||||
xq_loads(c, fx.min(fx.max(u, 0), xq_units - 1), lv)
|
||||
for u, lv in zip(xunits, xlive)
|
||||
]
|
||||
if const_expr(before_wqkv is not None):
|
||||
before_wqkv()
|
||||
kops = gemv_loads(
|
||||
c,
|
||||
a["wqkv"],
|
||||
a["wqkv_s"],
|
||||
HIDDEN,
|
||||
task * ROWS,
|
||||
kbeg=part * KSTEPS,
|
||||
nsteps=KSTEPS,
|
||||
pred=has_wqkv,
|
||||
)
|
||||
if const_expr(after_wqkv is not None):
|
||||
after_wqkv()
|
||||
if const_expr(xq_stage): # noqa: SIM102 (a build-time and a run-time test)
|
||||
if is_free:
|
||||
for i in range_constexpr(xq_per):
|
||||
stage_xq(
|
||||
c, fx.min(fx.max(xunits[i], 0), xq_units - 1), xq_regs[i], xlive[i]
|
||||
)
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if tid < xq_per:
|
||||
u = free + n_free * tid
|
||||
if u < xq_units:
|
||||
c["mb"].put(c["xrdy"], u, fx.Int32(1))
|
||||
if const_expr(not c.get("index", False)):
|
||||
# an index source's top-k comes after this launch: K2 builds its table
|
||||
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
|
||||
stage_kt(c, t)
|
||||
stamp(c, 1)
|
||||
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
|
||||
|
||||
# ---- wqkv_a: one (row tile, K half) a CTA, every token tile; the wq_b
|
||||
# weights issued behind the input rows' loads
|
||||
def wqb_weights():
|
||||
return [
|
||||
gemv_loads(c, a["wqb"], a["wqb_s"], Q_RANK, (wqb_task + BLOCKS * i) * ROWS)
|
||||
for i in range(wqb_per_cta)
|
||||
]
|
||||
|
||||
if const_expr(in_cta_quant):
|
||||
if has_wqkv:
|
||||
quant_x_half(c, xregs[0], tiles[0][1])
|
||||
bops = wqb_weights()
|
||||
if has_wqkv:
|
||||
stage_wqkv(c, task, part, kops, *tiles[0], has_wqkv)
|
||||
else:
|
||||
await_flags(c, c["xrdy"], s, lambda u: WQKV_KPARTS * u + part)
|
||||
xr = (
|
||||
c["x8"],
|
||||
c["x8s"],
|
||||
X_WORDS,
|
||||
part * KWORDS,
|
||||
KWORDS,
|
||||
X_GROUPS,
|
||||
part * KGROUPS,
|
||||
KGROUPS,
|
||||
)
|
||||
regs = rows_loads(c, *xr, tiles[0][0], tiles[0][1], pred=has_wqkv)
|
||||
if const_expr(len(tiles) == 1):
|
||||
bops = wqb_weights()
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, KWORDS, KGROUPS, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
t1, n1 = tiles[ti + 1]
|
||||
regs = rows_loads(c, *xr, t1, n1, pred=has_wqkv)
|
||||
if const_expr(ti + 2 == len(tiles)):
|
||||
bops = wqb_weights()
|
||||
stage_wqkv(c, task, part, kops, t0, n, has_wqkv)
|
||||
stamp(c, 2)
|
||||
|
||||
# ---- q / kv norms, KV insert
|
||||
for t in range(first_task(bid, QKV0), s, BLOCKS):
|
||||
stage_qkv(c, t)
|
||||
stamp(c, 3)
|
||||
|
||||
# ---- wq_b: every task's rows of a token tile at a time, the next
|
||||
# tile's rows in flight
|
||||
ropes = [wqb_rope(c, wqb_task, poss[ti], tiles[ti][1]) for ti in range(len(tiles))]
|
||||
await_flags(c, c["qrdy"], s, lambda u: u)
|
||||
qx = (c["qx8"], c["qx8s"], QX_WORDS, 0, QX_WORDS, Q_RANK // 32, 0, Q_RANK // 32)
|
||||
regs = rows_loads(c, *qx, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, QX_WORDS, Q_RANK // 32, c["qxl"], c["qxsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *qx, *tiles[ti + 1])
|
||||
stage_wqb(c, wqb_task, bops, t0, n, ropes[ti])
|
||||
stamp(c, 4)
|
||||
@@ -0,0 +1,234 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 GEMV of the mega kernel: 16 rows of a row-major e4m3 weight [N, K]
|
||||
(vLLM's loaded layout, E8M0 scales in 32x32 blocks [N / 32, K / 32]) against
|
||||
every token of a tile, its MXFP8 rows in LDS, on the gfx950 scaled MFMA.
|
||||
|
||||
A task may cover a window of K (``kbeg`` .. ``kbeg + nsteps`` 128-steps): a
|
||||
split-K part, its activations in LDS holding just that window. The weight
|
||||
operands are loaded first (all in flight, before the task waits on its
|
||||
activations); wave w takes the window's steps w, w + 8, ... (or, ``split``, a
|
||||
contiguous range of waves 0 .. split - 1), its 16 x 16 partial in ``red``,
|
||||
summed in wave order by ``row_sum``. Adapted from ATOM's
|
||||
``attn_pre.gemv_fp8_*``, which reads aiter's (16, 16)-preshuffled layout.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from .device import THREADS, WAVES, mfma_scaled, rsrc, tile_map, traced
|
||||
|
||||
# a token's LDS row of MXFP8 words / E8M0 codes, padded by 16 B so 16 rows read
|
||||
# by one MFMA operand do not share a bank group
|
||||
LDS_PAD = 4
|
||||
RED = WAVES * 64 * 4 # one task's accumulators in ``red`` (f32)
|
||||
|
||||
|
||||
def lds_row(words):
|
||||
return words + LDS_PAD
|
||||
|
||||
|
||||
def lds_at(u, row):
|
||||
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
|
||||
return u + u // row * LDS_PAD
|
||||
|
||||
|
||||
def _steps(c, nsteps, split, tiled):
|
||||
"""This wave's steps of the window: [(local step, live)]."""
|
||||
wave = c["wave"]
|
||||
if const_expr(split is None):
|
||||
per = (nsteps + WAVES - 1) // WAVES
|
||||
ragged = nsteps % WAVES != 0
|
||||
return [
|
||||
(wave + WAVES * i, (wave + WAVES * i < nsteps) if ragged else None)
|
||||
for i in range(per)
|
||||
]
|
||||
assert split <= WAVES and nsteps % split == 0
|
||||
per = nsteps // split
|
||||
part = wave % split if tiled else fx.min(wave, split - 1)
|
||||
return [(part * per + i, None) for i in range(per)]
|
||||
|
||||
|
||||
def _step_load(c, r_w, r_ws, k, row0, st, pred=None):
|
||||
"""Global step ``st``'s weight operand of rows row0 .. +16: two 16 B chunks a
|
||||
lane (K bytes 16 j .. and 64 + 16 j .. of the step, row row0 + lane % 16)
|
||||
and the E8M0 byte of its K block. ``pred`` false: masked (out of range: no
|
||||
memory traffic, zeros)."""
|
||||
lane = c["lane"]
|
||||
j = lane // 16
|
||||
row = row0 + lane % 16
|
||||
base = row * k + 128 * st + 16 * j
|
||||
lo = fx.Vector(bo.buffer_load(r_w, base // 4, vec_width=4, dtype=T.i32))
|
||||
hi = fx.Vector(bo.buffer_load(r_w, (base + 64) // 4, vec_width=4, dtype=T.i32))
|
||||
kg = k // 32
|
||||
wsi = (row0 // 32) * kg + st * 4 + j
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def gemv_loads(
|
||||
c, w, ws, k, row0, split=None, tiled=False, kbeg=0, nsteps=None, pred=None
|
||||
):
|
||||
"""The weight operands of rows row0 .. +16 (``tiled``: row0 + 16 (wave //
|
||||
split)) over steps kbeg .. kbeg + nsteps for this wave, all in flight
|
||||
(``pred`` false: masked off, no memory traffic)."""
|
||||
nsteps = k // 128 if nsteps is None else nsteps
|
||||
if const_expr(pred is None):
|
||||
r_w, r_ws = rsrc(w), rsrc(ws)
|
||||
else:
|
||||
# a masked-off CTA reads through a zero-size buffer: zeros, no traffic
|
||||
nb = pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
|
||||
r_w, r_ws = rsrc(w, nb), rsrc(ws, nb)
|
||||
if const_expr(tiled):
|
||||
row0 = row0 + 16 * (c["wave"] // split)
|
||||
return [
|
||||
_step_load(
|
||||
c,
|
||||
r_w,
|
||||
r_ws,
|
||||
k,
|
||||
row0,
|
||||
kbeg + (st if live is None else fx.min(st, nsteps - 1)),
|
||||
pred,
|
||||
)
|
||||
for st, live in _steps(c, nsteps, split, tiled)
|
||||
]
|
||||
|
||||
|
||||
def _step_mfma(c, nsteps, xl, xsl, st, ops, acc, live, rows):
|
||||
lane = c["lane"]
|
||||
lo, hi, sa = ops
|
||||
x_row, s_row = lds_row(nsteps * 32), lds_row(nsteps * 4)
|
||||
j = lane // 16
|
||||
col = fx.min(lane % 16, rows - 1)
|
||||
if const_expr(live is not None):
|
||||
st = fx.min(st, nsteps - 1)
|
||||
av = fx.Vector.from_elements(
|
||||
[lo[d] for d in range(4)] + [hi[d] for d in range(4)], fx.Int32
|
||||
)
|
||||
xlo = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xhi = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + 16 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xv = fx.Vector.from_elements(
|
||||
[xlo[d] for d in range(4)] + [xhi[d] for d in range(4)], fx.Int32
|
||||
)
|
||||
sb = fx.ptr_load(xsl + (col * s_row + st * 4 + j))
|
||||
if const_expr(live is not None):
|
||||
# a step past the end adds zero at unit scale
|
||||
av = fx.Vector.from_elements(
|
||||
[live.select(av[d], fx.Int32(0)) for d in range(8)], fx.Int32
|
||||
)
|
||||
sa = live.select(sa, fx.Int32(127))
|
||||
sb = live.select(sb, fx.Int32(127))
|
||||
return mfma_scaled(av, xv, acc, sa, sb)
|
||||
|
||||
|
||||
def gemv_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=16, nsteps=None):
|
||||
"""``gemv_loads``' operands against ``rows`` tokens' MXFP8 rows of the
|
||||
window in LDS (``xl`` words, ``xsl`` codes, rows ``lds_row`` of the window)
|
||||
-> ``red[wave][lane]`` (a 16 x 16 C: row 4 (lane // 16) + i, token lane % 16)."""
|
||||
lane, wave = c["lane"], c["wave"]
|
||||
nsteps = k // 128 if nsteps is None else nsteps
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i, (st, live) in enumerate(_steps(c, nsteps, split, tiled)):
|
||||
acc = _step_mfma(c, nsteps, xl, xsl, st, ops[i], acc, live, rows)
|
||||
if const_expr(split is not None and not tiled):
|
||||
acc = fx.Vector.from_elements(
|
||||
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
|
||||
fx.Float32,
|
||||
)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
|
||||
|
||||
def row_sum(red, r, t, waves=None):
|
||||
"""Row r, token t of the waves' accumulators, summed in wave order."""
|
||||
tot = fx.Float32(0.0)
|
||||
for w in waves if waves is not None else range(WAVES):
|
||||
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
|
||||
return tot
|
||||
|
||||
|
||||
def span_items(width, n):
|
||||
"""A [n, width] span of 16 B items: rows a round of THREADS threads, rounds."""
|
||||
wq = width // 4
|
||||
rpr = THREADS // wq
|
||||
assert rpr >= 1, width
|
||||
return wq, rpr, (n + rpr - 1) // rpr
|
||||
|
||||
|
||||
def rows_loads(
|
||||
c,
|
||||
words_base,
|
||||
codes_base,
|
||||
stride_w,
|
||||
off_w,
|
||||
words,
|
||||
stride_g,
|
||||
off_g,
|
||||
groups,
|
||||
t0,
|
||||
n,
|
||||
pred=None,
|
||||
):
|
||||
"""Tokens t0 .. t0 + n's published MXFP8 row slices -- ``words`` i32 from
|
||||
``off_w`` and ``groups`` codes (an i32 each) from ``off_g``, rows
|
||||
``stride_w`` / ``stride_g`` i32 apart -- loaded (issued, not waited on),
|
||||
16 B a lane-item: a thread one column, rows a round apart. Plain loads:
|
||||
their flags are seen, and nothing in the launch read these lines before.
|
||||
``pred`` false: masked off (zeros, no memory traffic)."""
|
||||
tid = c["tid"]
|
||||
nb = None if pred is None else pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
|
||||
out = []
|
||||
for base, stride, off, width in (
|
||||
(words_base, stride_w, off_w, words),
|
||||
(codes_base, stride_g, off_g, groups),
|
||||
):
|
||||
wq, rpr, rounds = span_items(width, n)
|
||||
row0, col = tile_map(tid, wq)
|
||||
a0 = col * 4 + off + t0 * stride
|
||||
r = rsrc(base, nb)
|
||||
regs = []
|
||||
for i in range(rounds):
|
||||
row = row0 + rpr * i
|
||||
if (i + 1) * rpr > n or rpr * wq < THREADS:
|
||||
row = fx.min(row, n - 1)
|
||||
regs.append(
|
||||
fx.Vector(
|
||||
bo.buffer_load(r, row * stride + a0, vec_width=4, dtype=T.i32)
|
||||
)
|
||||
)
|
||||
out.append(regs)
|
||||
return out
|
||||
|
||||
|
||||
@traced
|
||||
def _store_span(c, regs, width, dst, n):
|
||||
tid = c["tid"]
|
||||
wq, rpr, _ = span_items(width, n)
|
||||
row0, col = tile_map(tid, wq)
|
||||
lrow = lds_row(width)
|
||||
base = dst + (row0 * lrow + col * 4)
|
||||
for i in range_constexpr(len(regs)):
|
||||
if const_expr((i + 1) * rpr <= n and rpr * wq == THREADS):
|
||||
fx.ptr_store(regs[i], base + i * rpr * lrow)
|
||||
else:
|
||||
if (row0 + rpr * i < n) & (tid < rpr * wq):
|
||||
fx.ptr_store(regs[i], base + i * rpr * lrow)
|
||||
|
||||
|
||||
def rows_store(c, regs, words, groups, xl, xsl, n):
|
||||
"""``rows_loads``' registers into LDS rows 0 .. n (``lds_row`` strides)."""
|
||||
_store_span(c, regs[0], words, xl, n)
|
||||
_store_span(c, regs[1], groups, xsl, n)
|
||||
gpu.barrier()
|
||||
@@ -0,0 +1,173 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Geometry of the DeepSeek-V4.1 attention mega kernel: model widths, a TP
|
||||
rank's share of them, task counts and the scratch (mailbox) layout."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
HIDDEN = 5120
|
||||
Q_RANK = 1280
|
||||
KV_DIM = 512
|
||||
QKV_ROWS = Q_RANK + KV_DIM # 1792
|
||||
HEAD_DIM = 512
|
||||
NOPE = 448
|
||||
ROPE = 64
|
||||
HALF = ROPE // 2
|
||||
N_HEADS = 64
|
||||
O_GROUPS = 8
|
||||
O_RANK = 1024
|
||||
WINDOW = 128
|
||||
TOPK = 512
|
||||
EPS = 1e-20
|
||||
|
||||
ROWS = 16 # GEMV rows a task
|
||||
TILE = 16 # tokens a GEMV pass (an MFMA's N)
|
||||
HEAD_TILE = 16 # heads an attention MFMA takes
|
||||
WOA_ROWS = 32 # a wo_a task's rows: one MX group of its output
|
||||
MAX_TOKENS = 48
|
||||
|
||||
# KV record (V4 fp8_ds_mla): 576 B data rows (448 e4m3 | 64 bf16), then 8 B of
|
||||
# scales a token (7 UE8M0 + pad) after the block's rows
|
||||
DATA = 576
|
||||
SCALE_BYTES = 8
|
||||
|
||||
# attention: a split is 16 keys; a token has at most TOPK + WINDOW keys
|
||||
BK = 16
|
||||
SPLITS = 64 # split slots a (token, head tile): 1024 keys >= TOPK + WINDOW
|
||||
COLS = 32 # PV columns a group (one MX group of the output)
|
||||
GROUPS = HEAD_DIM // COLS # 16
|
||||
|
||||
PAIR = 8 # a mailbox (value, tag) pair
|
||||
|
||||
|
||||
def cdiv(a, b):
|
||||
return -(-a // b)
|
||||
|
||||
|
||||
def round_up(a, b):
|
||||
return cdiv(a, b) * b
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dims:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.tp in (2, 4), self.tp
|
||||
|
||||
@property
|
||||
def heads(self) -> int:
|
||||
return N_HEADS // self.tp
|
||||
|
||||
@property
|
||||
def head_tiles(self) -> int:
|
||||
return self.heads // HEAD_TILE
|
||||
|
||||
@property
|
||||
def groups(self) -> int:
|
||||
return O_GROUPS // self.tp
|
||||
|
||||
@property
|
||||
def group_k(self) -> int:
|
||||
"""wo_a's K a group: its eight heads' outputs."""
|
||||
return self.heads // self.groups * HEAD_DIM # 4096
|
||||
|
||||
@property
|
||||
def o_rows(self) -> int:
|
||||
return self.groups * O_RANK
|
||||
|
||||
@property
|
||||
def q_rows(self) -> int:
|
||||
return self.heads * HEAD_DIM
|
||||
|
||||
@property
|
||||
def wqb_tasks(self) -> int:
|
||||
return self.q_rows // ROWS
|
||||
|
||||
@property
|
||||
def woa_tasks(self) -> int:
|
||||
return self.o_rows // WOA_ROWS
|
||||
|
||||
@property
|
||||
def wob_tasks(self) -> int:
|
||||
return HIDDEN // ROWS # 320
|
||||
|
||||
|
||||
WQKV_TASKS = QKV_ROWS // ROWS # 112
|
||||
WQKV_KPARTS = 2 # wqkv's K split: 224 (row tile, K half) parts
|
||||
KEYS = TOPK + WINDOW # a token's attention keys at most
|
||||
|
||||
|
||||
def token_tiles(s):
|
||||
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
|
||||
|
||||
|
||||
def tile_rows(s):
|
||||
return min(s, TILE)
|
||||
|
||||
|
||||
def pair_layout(items, start=0, align=256):
|
||||
out, off = {}, start
|
||||
for name, pairs in items:
|
||||
off = round_up(off, align)
|
||||
out[name] = (off, pairs * PAIR)
|
||||
off += pairs * PAIR
|
||||
return out
|
||||
|
||||
|
||||
def byte_layout(items, start=0, align=256):
|
||||
out, off = {}, start
|
||||
for name, nbytes in items:
|
||||
off = round_up(off, align)
|
||||
out[name] = (off, nbytes)
|
||||
off += nbytes
|
||||
return out
|
||||
|
||||
|
||||
def front_scratch(s: int, d: Dims) -> dict:
|
||||
"""K1's scratch regions -> (byte offset, bytes): published data (plain
|
||||
words behind a flag) and mailbox pairs."""
|
||||
return byte_layout(
|
||||
[
|
||||
("x8", s * HIDDEN), # e4m3 words (S > 16: quantized once, shared)
|
||||
("x8s", s * HIDDEN // 32 * 4), # an i32 code a group
|
||||
("xrdy", WQKV_KPARTS * s * PAIR), # a flag a (token, K half)
|
||||
("qkvp", WQKV_KPARTS * s * QKV_ROWS * PAIR),
|
||||
("qx8", s * Q_RANK), # e4m3 words
|
||||
("qx8s", s * Q_RANK // 32 * 4), # an i32 code a group
|
||||
("qrdy", s * PAIR),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def back_scratch(s: int, d: Dims, start: int = 0) -> dict:
|
||||
"""K2's scratch regions, laid out past ``start``: published data (plain)
|
||||
and flag pairs."""
|
||||
ck = (
|
||||
64 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= 256 else 128
|
||||
) # back.chunk_keys
|
||||
nchunk = cdiv(KEYS, ck)
|
||||
return byte_layout(
|
||||
[
|
||||
("part", s * nchunk * d.heads * HEAD_DIM * 2), # bf16 partials
|
||||
("pstat", s * nchunk * d.heads * 8), # (max, sum) f32
|
||||
("srdy", s * nchunk * PAIR),
|
||||
("xo", s * d.q_rows), # e4m3 words
|
||||
("xos", s * d.q_rows // 32 * 4), # an i32 code a group
|
||||
("crdy", s * d.heads // 8 * PAIR),
|
||||
("x8b", s * d.o_rows),
|
||||
("x8bs", s * d.o_rows // 32 * 4),
|
||||
("ardy", d.woa_tasks * PAIR),
|
||||
],
|
||||
start=start,
|
||||
)
|
||||
|
||||
|
||||
def front_end(s: int, d: Dims) -> int:
|
||||
"""Where K2's regions start: past K1's, which share the launch tag."""
|
||||
return round_up(max(o + n for o, n in front_scratch(s, d).values()), 256)
|
||||
|
||||
|
||||
def scratch_bytes(s: int, d: Dims) -> int:
|
||||
b = back_scratch(s, d, start=front_end(s, d))
|
||||
return round_up(max(o + n for o, n in b.values()), 4096)
|
||||
@@ -0,0 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/__init__.py
|
||||
"""The mono kernels' shared mechanisms: ``ops`` (wave / math primitives, the TP
|
||||
group's peer buffers), ``sync`` (the tagged mailbox), ``mx`` (FP8 / MX
|
||||
arithmetic), ``plan`` (execution model, layouts, build keys) and
|
||||
``peer_memory`` (symmetric peer buffers)."""
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/mx.py
|
||||
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
|
||||
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
|
||||
tile's operand loads and the scaled MFMA that takes them.
|
||||
|
||||
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
|
||||
|
||||
FP8_MAX = 448.0
|
||||
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
|
||||
FP8, FP4 = 0, 4
|
||||
UNIT_SCALE = 127 # the E8M0 code of 1.0
|
||||
|
||||
|
||||
def clamp_fp8(v):
|
||||
"""``v`` clamped to the E4M3 range."""
|
||||
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
||||
|
||||
|
||||
def code_ceil(v):
|
||||
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
|
||||
clamped to [1, 254]: the codes a scale may take."""
|
||||
bits = v.bitcast(fx.Int32)
|
||||
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
||||
fx.Int32(1), fx.Int32(0)
|
||||
)
|
||||
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
|
||||
|
||||
|
||||
def rcp_pow2(code):
|
||||
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
|
||||
bit for bit, a multiply where the division is a dozen instructions."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def scale_index(row, col, cols):
|
||||
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
|
||||
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
|
||||
r32, a, b = row // 32, (row // 16) % 2, row % 16
|
||||
c8, d, e = col // 8, (col // 4) % 2, col % 4
|
||||
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
|
||||
|
||||
|
||||
def fp4_tile_load(r_w, base, rg, k, st, at):
|
||||
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
|
||||
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
|
||||
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
|
||||
Nontemporal: a weight a step reads once."""
|
||||
return fx.Vector(
|
||||
bo.buffer_load(
|
||||
r_w,
|
||||
base + rg * (2 * k) + st * 256 + at,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_NT,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
|
||||
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
|
||||
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
|
||||
byte ``sb_byte``)."""
|
||||
return fx.Vector(
|
||||
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
||||
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,300 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/ops.py
|
||||
# atom/mono/device/ranks.py
|
||||
# atom/mono/device/stamps.py
|
||||
"""Device-side primitives of the mono kernels: wave reductions and shuffles, the
|
||||
approximate hardware math, bf16 / fp8 packing, buffer resources, the
|
||||
execution-model constants and the kernel symbol naming.
|
||||
|
||||
Ported from the FlyDSL GLM-5 layer kernel (ROCm/FlyDSL branch
|
||||
``glm5-mla-moe-layer-monokernel`` @ 1e28fd8a,
|
||||
``kernels/mla_moe_layer/shared_reuse_moe_kernel.py``). Only published flydsl APIs
|
||||
and aiter's vendored ``buffer_ops`` / ``dpp_utils`` are used.
|
||||
|
||||
Every function but ``kernel_symbol`` is traced inside a ``@flyc.kernel`` body.
|
||||
|
||||
Raw boundaries kept, for want of an equivalent on the pinned FlyDSL:
|
||||
``buffer_ops`` loads / stores (the kernels take raw Int64 addresses, no tensors
|
||||
for ``make_buffer_tensor``); raw MFMA intrinsics (hand-laid operands: permlane
|
||||
transposes, fp8 packing, MXFP4 scales); the LLVM intrinsics here (``expect``,
|
||||
``s_memrealtime``, the approximate rsq / rcp / exp2) and the struct unpacking of
|
||||
``permlane*_swap``.
|
||||
|
||||
Also the TP group seen from inside a kernel (every rank's peer buffer, an
|
||||
all-reduce's rank-ordered sum) and a timeline build's phase stamps."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
|
||||
from flydsl._mlir.dialects import llvm
|
||||
from flydsl.compiler.ast_rewriter import ASTRewriter
|
||||
from flydsl.expr import const_expr, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES
|
||||
|
||||
# ---------------------------------------------------------------- primitives
|
||||
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16. SC1 alone is device
|
||||
# scope (past the per-XCD caches); SC0 | SC1 is system scope (peers over XGMI).
|
||||
CM_DEV = 16
|
||||
|
||||
|
||||
CM_SYS = 17
|
||||
|
||||
|
||||
# NT = 2: nontemporal. A weight a layer reads once streams past the caches it
|
||||
# would only evict the step's reused data from (the MoE at 48 rows: 143 -> 133 us)
|
||||
CM_NT = 2
|
||||
|
||||
|
||||
def rsrc(addr, nbytes=None):
|
||||
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
|
||||
|
||||
|
||||
def permlane_swap(off, x, y):
|
||||
"""v_permlane{32,16}_swap on registers (x, y): off 32 trades x's upper-half
|
||||
lanes with y's lower-half lanes; off 16 trades x's odd 16-lane rows with y's
|
||||
even rows (each within its half). -> (x', y')."""
|
||||
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
|
||||
pr = swap(
|
||||
llvm.StructType.get_literal([T.i32, T.i32]),
|
||||
as_ir_value(fx.Int32(x)),
|
||||
as_ir_value(fx.Int32(y)),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
|
||||
|
||||
|
||||
def lane_gather(v, src_lane):
|
||||
"""Lane ``src_lane``'s value of v, a source lane a lane (ds_bpermute)."""
|
||||
return fx.Int32(
|
||||
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
|
||||
)
|
||||
|
||||
|
||||
def memrealtime():
|
||||
"""s_memrealtime: the 100 MHz device clock (no FlyDSL wrapper)."""
|
||||
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
|
||||
|
||||
|
||||
def fresh(v):
|
||||
"""``v`` (a per-lane Int32) through an opaque move, here: the lane math built
|
||||
on it is computed at its use, where LLVM would otherwise hoist it out of a task
|
||||
loop or CSE it across stages into registers held for the whole kernel (spills,
|
||||
and a kernel with scratch is refused: ``runtime.widths``)."""
|
||||
return fx.Int32(
|
||||
llvm.inline_asm(
|
||||
T.i32,
|
||||
[fx.Int32(v).ir_value()],
|
||||
"v_mov_b32 $0, $1",
|
||||
"=v,v",
|
||||
has_side_effects=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def uniform(v):
|
||||
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
|
||||
|
||||
|
||||
def _hw_f32(name, x):
|
||||
return fx.Float32(
|
||||
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
|
||||
)
|
||||
|
||||
|
||||
def hw_rsq(x):
|
||||
"""v_rsq_f32 (~1 ulp, no libm range fixups)."""
|
||||
return _hw_f32("llvm.amdgcn.rsq.f32", x)
|
||||
|
||||
|
||||
def hw_rcp(x):
|
||||
return _hw_f32("llvm.amdgcn.rcp.f32", x)
|
||||
|
||||
|
||||
def hw_exp2(x):
|
||||
return _hw_f32("llvm.amdgcn.exp2.f32", x)
|
||||
|
||||
|
||||
def div_rn(x, d, r):
|
||||
"""``x / d`` rounded to nearest, bit-identical to the IEEE division, given
|
||||
``r = 1.0 / d`` from one true division per divisor (Markstein: a quotient
|
||||
within 1 ulp plus its exact fma residual times RN(1/d) rounds correctly)."""
|
||||
q = x * r
|
||||
return fx.Float32(fmath.fma(fx.Float32(fmath.fma(-q, d, x)), r, q))
|
||||
|
||||
|
||||
def xshfl(v, off):
|
||||
"""Value of lane ``lane ^ off``: offsets 32 / 16 via v_permlane*_swap, in-row
|
||||
offsets via DPP (VALU latency, not LDS). Powers of two only: any other
|
||||
offset would take the lane ^ 1 path."""
|
||||
assert off in (1, 2, 4, 8, 16, 32), off
|
||||
if off >= 16:
|
||||
return v.shuffle_xor(off, 64)
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
if off == 8:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
|
||||
elif off == 4:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
|
||||
elif off == 2:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
|
||||
else:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
|
||||
return y.bitcast(fx.Float32) if is_f else y
|
||||
|
||||
|
||||
def xred(v, off, op):
|
||||
"""op(v, lane ^ off) for a symmetric op; 32 / 16 take both halves of one swap."""
|
||||
if off < 16:
|
||||
return op(v, xshfl(v, off))
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
a, b = permlane_swap(off, x, x)
|
||||
if is_f:
|
||||
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
|
||||
return op(type(v)(a), type(v)(b))
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return a + b
|
||||
|
||||
|
||||
def butterfly(v, offsets, op=_add):
|
||||
"""Butterfly reduction over the given xor offsets, in the given order.
|
||||
|
||||
The order is the numerical contract: the aiter per-head norm reduces 32
|
||||
lanes with offsets 16, 8, 4, 2, 1, and matching it keeps rstd bit-exact."""
|
||||
for off in offsets:
|
||||
v = xred(v, off, op)
|
||||
return v
|
||||
|
||||
|
||||
def traced(fn):
|
||||
"""Apply flydsl's AST rewriting to a helper, so its ``while`` / ``if`` over
|
||||
traced values lower to scf ops. Kernels rewrite only their own body."""
|
||||
return ASTRewriter.transform(fn)
|
||||
|
||||
|
||||
LOAD_BATCH = 4 # rounds of a strided loop whose loads go out together
|
||||
|
||||
|
||||
@traced
|
||||
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
|
||||
"""A CTA's strided loop over elements v = tid + THREADS i < n (n a Python
|
||||
int): ``load(v)`` of ``batch`` rounds issued before any ``use(v, loaded)``.
|
||||
A round whose ``use`` stores to global memory keeps the next round's loads
|
||||
behind it (they may alias), so the rounds' loads would go out in series. A
|
||||
round past n loads element n - 1 (in range) and uses nothing."""
|
||||
rounds = -(-n // THREADS)
|
||||
for b0 in range_constexpr(0, rounds, batch):
|
||||
idx = range(b0, min(b0 + batch, rounds))
|
||||
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
|
||||
for i, g in zip(idx, got):
|
||||
v = tid + THREADS * i
|
||||
if v < n:
|
||||
use(v, g)
|
||||
|
||||
|
||||
def mfma_bf16(a, b, c):
|
||||
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
|
||||
|
||||
|
||||
def row_sum(red, r, t, waves=None):
|
||||
"""Row r, token t of the waves' 16x16 MFMA accumulators (wave w's at ``red``
|
||||
+ (w * 64 + lane) * 4), summed in wave order from 0 (``waves``: which, in
|
||||
order; all of them by default)."""
|
||||
tot = fx.Float32(0.0)
|
||||
for w in waves if waves is not None else range(WAVES):
|
||||
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
|
||||
return tot
|
||||
|
||||
|
||||
def ballot(pred):
|
||||
"""The wave's lanes where ``pred`` holds, a bit a lane."""
|
||||
return fx.Int64(rocdl.ballot(T.i64, as_ir_value(pred)))
|
||||
|
||||
|
||||
def popcount(x):
|
||||
return fx.Int32(fx.ctpop(x))
|
||||
|
||||
|
||||
def lanes_below(bal, lane):
|
||||
"""How many of the lanes in ballot ``bal`` are below ``lane``."""
|
||||
return popcount(bal & ((fx.Int64(1) << fx.Int64(lane)) - 1))
|
||||
|
||||
|
||||
def load_ptr64(table, i):
|
||||
"""Entry ``i`` of an i64 table in memory, wave-uniform (each half through
|
||||
``uniform``): a peer's buffer base, a kernel argument block's pointer."""
|
||||
pw = fx.Vector(bo.buffer_load(rsrc(table), i * 2, vec_width=2, dtype=T.i32))
|
||||
return (fx.Int64(uniform(pw[1])) << 32) | fx.Int64(fx.Uint32(uniform(pw[0])))
|
||||
|
||||
|
||||
def bf16_round(a):
|
||||
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
|
||||
|
||||
|
||||
def bf16_pair(a, b):
|
||||
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
|
||||
return (
|
||||
fx.Vector.from_elements([a, b], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Float32)[0]
|
||||
)
|
||||
|
||||
|
||||
def bf_lo(w):
|
||||
"""The f32 of a packed bf16 pair word's low half."""
|
||||
return (w << 16).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf_hi(w):
|
||||
"""The f32 of a packed bf16 pair word's high half."""
|
||||
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf2_f32(w):
|
||||
"""Packed bf16 pair word -> (f32 low, f32 high)."""
|
||||
return bf_lo(w), bf_hi(w)
|
||||
|
||||
|
||||
def fp8_pack4(a, b, c, d):
|
||||
"""Four f32 (already in range) -> one dword of four E4M3 bytes, RNE."""
|
||||
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
|
||||
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP ranks
|
||||
def peer_bases(peers, tp):
|
||||
"""Every rank's peer buffer base from the ``peers`` table (an i64 a rank),
|
||||
loaded once: a put's address must not wait on a load of its own."""
|
||||
return [load_ptr64(peers, p) for p in range(tp)]
|
||||
|
||||
|
||||
def sum_partials(poll, own, pair_of, tp):
|
||||
"""The ``tp`` ranks' bf16 pair partials at ``pair_of(src)`` of the mailbox
|
||||
region ``own``, summed in fp32 in rank order 0 .. tp - 1: one poll batch ->
|
||||
(lo sum, hi sum). Every rank sums in the same order, so they agree."""
|
||||
ws = poll([(own, pair_of(src), 1) for src in range(tp)])
|
||||
acc0, acc1 = bf2_f32(ws[0][0])
|
||||
for src in range_constexpr(1, tp):
|
||||
lo, hi = bf2_f32(ws[src][0])
|
||||
acc0 = acc0 + lo
|
||||
acc1 = acc1 + hi
|
||||
return acc0, acc1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- timeline stamps
|
||||
def stamp(on, tls, tid, k):
|
||||
# compile-time gate outside, traced condition inside: they cannot be one `and`
|
||||
if const_expr(on): # noqa: SIM102
|
||||
if tid == 0:
|
||||
fx.ptr_store(memrealtime(), tls + k)
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/runtime/consensus.py
|
||||
# atom/mono/runtime/peer_memory.py
|
||||
"""Symmetric peer memory for the in-kernel TP all-reduces, and the agreement every
|
||||
rank must reach before anything collective (a rank that turned the mono layer
|
||||
off alone would leave the others spinning on the GPU)."""
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from aiter.ops.flydsl.quick_allreduce_int4_ipc import UncachedIpcHeap
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP agreement
|
||||
class MonoUnsupported(Exception):
|
||||
"""The loaded model or runtime configuration is outside what mono serves."""
|
||||
|
||||
|
||||
def tp_agree(ok: bool, group) -> bool:
|
||||
"""True when every rank of ``group`` passed ``ok`` true. Every rank must call
|
||||
it at the same point; ``group`` is a CPU (gloo) process group, or None for a
|
||||
single rank."""
|
||||
if group is None:
|
||||
return ok
|
||||
flag = torch.tensor([int(ok)], dtype=torch.int32)
|
||||
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group)
|
||||
return bool(flag.item())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- peer buffers
|
||||
class _DeviceBytes:
|
||||
"""``__cuda_array_interface__`` of ``nbytes`` bytes at a raw device address, for a
|
||||
non-owning torch view."""
|
||||
|
||||
def __init__(self, ptr: int, nbytes: int):
|
||||
self.__cuda_array_interface__ = {
|
||||
"shape": (nbytes,),
|
||||
"typestr": "|u1",
|
||||
"data": (ptr, False),
|
||||
"version": 3,
|
||||
}
|
||||
|
||||
|
||||
class PeerBuffer:
|
||||
"""One uncached buffer per rank, every rank holding every peer's address
|
||||
(``addresses``, int64, indexed by TP rank). What lives where, and why no rank
|
||||
overwrites a peer's unread data, is the model's layout's to say.
|
||||
|
||||
The constructor is collective over ``group`` (the handle exchange, then an
|
||||
agreement that every rank mapped every peer): every rank must reach it, so a
|
||||
runner reaches it only after ``tp_agree``. A failed mapping on any rank raises
|
||||
``MonoUnsupported`` on all of them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
nbytes: int,
|
||||
group,
|
||||
rank: int,
|
||||
npes: int,
|
||||
device: torch.device,
|
||||
):
|
||||
self.local = UncachedIpcHeap.alloc_uncached(nbytes)
|
||||
# a view, not an owner: close() frees the memory under it
|
||||
self.bytes = torch.as_tensor(_DeviceBytes(self.local, nbytes), device=device)
|
||||
self.rank = rank
|
||||
self._opened: list[int] = []
|
||||
addresses = [self.local]
|
||||
if npes > 1:
|
||||
handles = UncachedIpcHeap.gather_object_list_via_broadcast(
|
||||
group, UncachedIpcHeap.get_mem_handle_bytes(self.local)
|
||||
)
|
||||
addresses, failure = [], None
|
||||
try:
|
||||
for peer, handle in enumerate(handles):
|
||||
if peer == rank:
|
||||
addresses.append(self.local)
|
||||
else:
|
||||
base = UncachedIpcHeap.open_mem_handle(handle)
|
||||
self._opened.append(base)
|
||||
addresses.append(base)
|
||||
except RuntimeError as err: # every rank must still reach the agreement
|
||||
failure = err
|
||||
if not tp_agree(failure is None, group):
|
||||
self.close()
|
||||
raise MonoUnsupported(
|
||||
"peer memory handshake failed on "
|
||||
f"{'this' if failure else 'another'}"
|
||||
" TP rank"
|
||||
) from failure
|
||||
self.addresses = torch.tensor(addresses, dtype=torch.int64, device=device)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the peer mappings and free the local buffer; idempotent. Only once no
|
||||
kernel of any rank can still touch it (the runner is being dropped)."""
|
||||
opened, self._opened = self._opened, []
|
||||
for base in opened:
|
||||
UncachedIpcHeap.close_mem_handle(base)
|
||||
if self.local:
|
||||
self.bytes = None
|
||||
UncachedIpcHeap.free_device_mem(self.local)
|
||||
self.local = 0
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/plan/execution.py
|
||||
# atom/mono/plan/layout.py
|
||||
# atom/mono/plan/arith.py
|
||||
# atom/mono/plan/shard.py
|
||||
# atom/mono/plan/build_key.py
|
||||
"""Plain-Python planning of the mono kernels: the execution model (one CTA a CU,
|
||||
all resident), mailbox layouts, arithmetic that runs on ints and traced values
|
||||
alike, TP shards, and build keys (every build parameter, and this package's
|
||||
sources, in FlyDSL's JIT cache key)."""
|
||||
|
||||
import enum
|
||||
import functools
|
||||
import hashlib
|
||||
from dataclasses import dataclass, fields
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------- execution model
|
||||
BLOCKS = 256
|
||||
THREADS = 512
|
||||
WAVES = THREADS // 64
|
||||
|
||||
|
||||
def first_task(bid, base):
|
||||
"""CTA ``bid``'s first task of a stage placed from CTA ``base`` (any count of
|
||||
CTAs past 0: placements that add task counts run past the grid), then every
|
||||
BLOCKS-th."""
|
||||
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- mailbox layouts
|
||||
PAIR_BYTES = 8 # a (value, tag) pair
|
||||
|
||||
|
||||
def pair_layout(items, align: int = PAIR_BYTES, start: int = 0) -> dict:
|
||||
"""``items`` (name, pairs) laid out in order: name -> (byte offset, bytes),
|
||||
each region from an ``align``-byte boundary at or past ``start``."""
|
||||
out, off = {}, start
|
||||
for name, pairs in items:
|
||||
off = (off + align - 1) // align * align
|
||||
out[name] = (off, pairs * PAIR_BYTES)
|
||||
off += pairs * PAIR_BYTES
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- arithmetic
|
||||
def sel(pred, a, b):
|
||||
"""A if pred else b: a conditional on Python values, ``select`` on traced
|
||||
ones (rewrapped in the traced operand's type)."""
|
||||
if isinstance(pred, bool):
|
||||
return a if pred else b
|
||||
out = pred.select(a, b)
|
||||
traced = [x for x in (a, b) if not isinstance(x, int)]
|
||||
return type(traced[0])(out) if traced else out
|
||||
|
||||
|
||||
def imin(a, b):
|
||||
return sel(a < b, a, b)
|
||||
|
||||
|
||||
def i32(x):
|
||||
"""``x``, asserted inside the device's Int32 range when it is a Python int."""
|
||||
if isinstance(x, int):
|
||||
assert -(2**31) <= x < 2**31, x
|
||||
return x
|
||||
|
||||
|
||||
def cdiv(a, b):
|
||||
"""ceil(a / b) for a >= 0, b > 0."""
|
||||
return i32((a + b - 1) // b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP shards
|
||||
class ShardError(ValueError):
|
||||
"""A width the TP size does not divide, or a TP size the model refuses."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Shard:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tp < 1:
|
||||
raise ShardError(f"TP {self.tp}")
|
||||
|
||||
def split(self, full: int, what: str) -> int:
|
||||
"""``full`` / tp, refusing a remainder: a rank's share of ``what``."""
|
||||
if full % self.tp:
|
||||
raise ShardError(f"{what} {full} is not divisible by TP {self.tp}")
|
||||
return full // self.tp
|
||||
|
||||
|
||||
def padded(n: int, align: int) -> int:
|
||||
return cdiv(n, align) * align
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- build keys
|
||||
_SCALARS = (int, float, bool, str, enum.Enum)
|
||||
_ROOT = Path(__file__).resolve().parents[1] # the mono package
|
||||
|
||||
|
||||
@functools.cache
|
||||
def source_digest(*paths: str) -> str:
|
||||
"""sha256 of every ``.py`` file under ``paths`` (files or directories,
|
||||
relative to the mono package), each file's path and contents: one model's
|
||||
kernel sources, so an edit to another model's rebuilds none of these."""
|
||||
h = hashlib.sha256()
|
||||
for rel in paths:
|
||||
root = _ROOT / rel
|
||||
files = [root] if root.is_file() else sorted(root.rglob("*.py"))
|
||||
if not files:
|
||||
raise FileNotFoundError(f"no kernel sources at {root}")
|
||||
for f in files:
|
||||
h.update(str(f.relative_to(_ROOT)).encode())
|
||||
h.update(f.read_bytes())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def key_tuple(key, sources: str) -> tuple:
|
||||
"""``key``'s (field name, value) pairs, every value a scalar, then
|
||||
("sources", ``sources``): the model's ``source_digest``."""
|
||||
out = []
|
||||
for f in fields(key):
|
||||
value = getattr(key, f.name)
|
||||
if not isinstance(value, _SCALARS):
|
||||
raise TypeError(
|
||||
f"{type(key).__name__}.{f.name} = {value!r}: a build parameter must be"
|
||||
" a scalar to reach the JIT cache key"
|
||||
)
|
||||
out.append((f.name, value))
|
||||
out.append(("sources", sources))
|
||||
return tuple(out)
|
||||
@@ -0,0 +1,177 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/sync.py
|
||||
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
|
||||
between GPUs (system scope), every address a named region (``Addr``)."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
CM_DEV,
|
||||
CM_SYS,
|
||||
bf16_pair,
|
||||
rsrc,
|
||||
traced,
|
||||
)
|
||||
|
||||
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
|
||||
|
||||
|
||||
@traced
|
||||
def _spin(load_all, pending):
|
||||
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
|
||||
loop keeps the loads from being hoisted."""
|
||||
v = load_all()
|
||||
while pending(v):
|
||||
rocdl.s_sleep(1)
|
||||
v = load_all()
|
||||
return v
|
||||
|
||||
|
||||
class Space(Enum):
|
||||
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
|
||||
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Addr:
|
||||
"""A mailbox region's address: the region's name and space, and the traced
|
||||
address. The space sets the hand-off's scope: device for SCRATCH, system for
|
||||
PEER."""
|
||||
|
||||
region: str
|
||||
space: Space
|
||||
value: object
|
||||
|
||||
|
||||
def sreg(base, offset, region):
|
||||
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
|
||||
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
|
||||
|
||||
|
||||
def preg(base, offset, region):
|
||||
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
|
||||
``base``."""
|
||||
return Addr(region, Space.PEER, base + fx.Int64(offset))
|
||||
|
||||
|
||||
# the store / load cache policy of a hand-off in each space
|
||||
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
|
||||
|
||||
|
||||
class Mailbox:
|
||||
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
|
||||
|
||||
Every 32-bit value is stored next to this launch's tag; a consumer
|
||||
polls the payload until all tags match, so a hand-off costs one round trip
|
||||
(no store drain, no separate flag). The mono layer's launches tag their
|
||||
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
|
||||
never carries a live tag. Every address is an ``Addr``: its space picks the
|
||||
scope.
|
||||
"""
|
||||
|
||||
def __init__(self, tag):
|
||||
self.tag = tag
|
||||
|
||||
def put(self, a, i, v):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
||||
rsrc(a.value),
|
||||
i * 2,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def put_bf(self, a, i, vs):
|
||||
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
|
||||
store."""
|
||||
words = []
|
||||
for j in range_constexpr(len(vs) // 2):
|
||||
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(a.value),
|
||||
i,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def put_words(self, a, i, words):
|
||||
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
|
||||
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
|
||||
not kept."""
|
||||
vals = []
|
||||
for w in words:
|
||||
vals += [fx.Int32(w), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(vals, fx.Int32),
|
||||
rsrc(a.value),
|
||||
i * 2,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def poll(self, specs, batch=POLL_MAX):
|
||||
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
|
||||
all of one space.
|
||||
|
||||
All pairs are re-loaded together while any tag is stale, so a batch costs
|
||||
one round trip after its last producer lands. Returns the value words."""
|
||||
if const_expr(len(specs) == 0):
|
||||
return []
|
||||
if const_expr(len(specs) > batch):
|
||||
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
||||
spaces = {a.space for a, _, _ in specs}
|
||||
assert len(spaces) == 1, f"one poll batch spans {spaces}"
|
||||
cm = _SCOPE_CM[spaces.pop()]
|
||||
bases = [a.value for a, _, _ in specs]
|
||||
tag = self.tag
|
||||
|
||||
def load_all():
|
||||
words = []
|
||||
for b, (_, i, n) in zip(bases, specs):
|
||||
w = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(b),
|
||||
fx.Int32(i) * 2,
|
||||
vec_width=2 * n,
|
||||
dtype=T.i32,
|
||||
cache_modifier=cm,
|
||||
)
|
||||
)
|
||||
words += [w[e] for e in range(2 * n)]
|
||||
return fx.Vector.from_elements(words, fx.Int32)
|
||||
|
||||
nw = sum(2 * n for _, _, n in specs)
|
||||
|
||||
def pending(v):
|
||||
bad = v[1] != tag
|
||||
for e in range_constexpr(3, nw, 2):
|
||||
bad = bad | (v[e] != tag)
|
||||
return bad
|
||||
|
||||
v = _spin(load_all, pending)
|
||||
outs, e = [], 0
|
||||
for _, _, n in specs:
|
||||
outs.append([v[e + 2 * q] for q in range(n)])
|
||||
e += 2 * n
|
||||
return outs
|
||||
|
||||
|
||||
@traced
|
||||
def publish(put, a, i, v, who):
|
||||
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
|
||||
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
|
||||
not overtake the data it announces, and a workgroup-scope release fence does
|
||||
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
|
||||
mode: the workgroup's waves share one CU, the readers here do not)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if who:
|
||||
put(a, i, v)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,464 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1-Flash mono decode layer: host side.
|
||||
|
||||
One ``DSV41MonoLayer`` a TP rank runs every mono layer of a decode step (M <= 48
|
||||
rows): two persistent launches a layer (``layer``) from the layer's
|
||||
inputs at the attention seam to its outputs at the next one -- the MoE's
|
||||
reduced output, the residual after the FFN seam and that seam's mixes -- with
|
||||
both TP all-reduces inside the kernels (symmetric peer memory). A layer whose
|
||||
attention vLLM runs (``ffn``) takes one launch from its unreduced attention
|
||||
output on. Every argument is a device pointer: the launches can be captured in a
|
||||
HIP graph. The kernels move their mailbox epoch on themselves; every rank runs
|
||||
the same launches, so the ranks' epochs agree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
|
||||
from .common.plan import BLOCKS
|
||||
from .layer import (
|
||||
EPOCH_WORDS,
|
||||
MAX_TOKENS,
|
||||
MonoBuild,
|
||||
build_mono_ffn,
|
||||
build_mono_k1,
|
||||
build_mono_k2,
|
||||
peer_half_bytes,
|
||||
scratch_bytes,
|
||||
)
|
||||
from .stages.dims import Dims as MoeDims
|
||||
from .stages.moe_shape import EXPERTS
|
||||
|
||||
__all__ = ["BLOCKS", "MAX_TOKENS", "AttnWeights", "DSV41MonoLayer", "MonoLayerWeights"]
|
||||
|
||||
_LOG2E = 1.4426950408889634
|
||||
HC = 4
|
||||
|
||||
|
||||
def _ensure_writable_flydsl_cache() -> None:
|
||||
"""Aiter points FlyDSL at its bundled, read-only cache; new kernels need a
|
||||
writable one."""
|
||||
cur = os.environ.get("FLYDSL_RUNTIME_CACHE_DIR")
|
||||
if cur and os.access(cur, os.W_OK):
|
||||
return
|
||||
path = Path.home() / ".flydsl" / "cache"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = str(path)
|
||||
|
||||
|
||||
def _check_tensors(owner, tag: str, want: dict) -> None:
|
||||
"""Each named tensor of ``owner`` has the (shape, dtype) the kernels read,
|
||||
contiguous: their buffer loads are unbounded, so a mismatch would read
|
||||
garbage rather than fault."""
|
||||
for name, (shape, dtype) in want.items():
|
||||
t = getattr(owner, name)
|
||||
assert tuple(t.shape) == shape and t.dtype == dtype and t.is_contiguous(), (
|
||||
f"{tag} {name}: {tuple(t.shape)} {t.dtype}, want {shape} {dtype}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttnWeights:
|
||||
"""One layer's attention tensors on this rank, in vLLM's loaded layout."""
|
||||
|
||||
layer_id: int
|
||||
wqkv: torch.Tensor # [1792, 5120] e4m3: [wq_a; wkv]
|
||||
wqkv_scale: torch.Tensor # [56, 160] uint8 E8M0 (32 x 32 blocks)
|
||||
q_norm: torch.Tensor # [1280] bf16
|
||||
kv_norm: torch.Tensor # [512] bf16
|
||||
wq_b: torch.Tensor # [H * 512, 1280] e4m3
|
||||
wq_b_scale: torch.Tensor # [H * 16, 40] uint8
|
||||
wo_a: torch.Tensor # [G * 1024, 4096] e4m3
|
||||
wo_a_scale: torch.Tensor # [G * 32, 128] uint8
|
||||
wo_b: torch.Tensor # [5120, G * 1024] e4m3
|
||||
wo_b_scale: torch.Tensor # [160, G * 32] uint8
|
||||
attn_sink: torch.Tensor # [>= H] f32
|
||||
cos_sin: torch.Tensor # [positions, 64] f32 (the layer's rope table)
|
||||
ratio: int # 0 / 1 / 2: the layer's compress ratio
|
||||
|
||||
def check(self, d: Dims) -> None:
|
||||
h, g = d.heads, d.groups
|
||||
want = {
|
||||
"wqkv": ((1792, HIDDEN), torch.float8_e4m3fn),
|
||||
"wqkv_scale": ((56, HIDDEN // 32), torch.uint8),
|
||||
"wq_b": ((h * HEAD_DIM, 1280), torch.float8_e4m3fn),
|
||||
"wq_b_scale": ((h * HEAD_DIM // 32, 40), torch.uint8),
|
||||
"wo_a": ((g * 1024, d.group_k), torch.float8_e4m3fn),
|
||||
"wo_a_scale": ((g * 32, d.group_k // 32), torch.uint8),
|
||||
"wo_b": ((HIDDEN, g * 1024), torch.float8_e4m3fn),
|
||||
"wo_b_scale": ((HIDDEN // 32, g * 32), torch.uint8),
|
||||
}
|
||||
_check_tensors(self, f"layer {self.layer_id}", want)
|
||||
assert self.cos_sin.dtype == torch.float32 and self.cos_sin.shape[-1] == 64
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonoLayerWeights:
|
||||
"""One layer's tensors on this rank, in vLLM's loaded layout (``attn``:
|
||||
None for a layer whose attention vLLM runs)."""
|
||||
|
||||
attn: AttnWeights | None
|
||||
hc_attn_fn: torch.Tensor # [24, 4 * 5120] f32
|
||||
hc_attn_scale: torch.Tensor # [3] f32
|
||||
hc_attn_base: torch.Tensor # [24] f32
|
||||
attn_norm: torch.Tensor # [5120] bf16
|
||||
hc_ffn_fn: torch.Tensor
|
||||
hc_ffn_scale: torch.Tensor
|
||||
hc_ffn_base: torch.Tensor
|
||||
ffn_norm: torch.Tensor
|
||||
gate_w: torch.Tensor # [384, 5120] bf16
|
||||
bias: torch.Tensor # [384] f32 (e_score_correction_bias)
|
||||
w13: torch.Tensor # [384, 2 inter, 2560] fp4x2, aiter (16, 16) shuffle
|
||||
w13_s: torch.Tensor # its scales, aiter shuffle_scale order
|
||||
w2: torch.Tensor # [384, 5120, inter / 2] fp4x2
|
||||
w2_s: torch.Tensor
|
||||
sgu: torch.Tensor # shared gate_up [2 inter, 5120] e4m3, row-major
|
||||
sgu_s: torch.Tensor # [2 inter / 32, 160] E8M0
|
||||
sw2: torch.Tensor # shared down [5120, inter] e4m3
|
||||
sw2_s: torch.Tensor
|
||||
|
||||
def check(self, tp: int) -> None:
|
||||
"""The MoE's tensors: 384 experts, TP-sharded intermediates (the routed
|
||||
one padded as the loader pads it), AITER's A8W4 layout, 32 x 32 E8M0
|
||||
blocks for the shared expert."""
|
||||
d, fp4, e4m3, u8 = (
|
||||
MoeDims(tp),
|
||||
torch.float4_e2m1fn_x2,
|
||||
torch.float8_e4m3fn,
|
||||
torch.uint8,
|
||||
)
|
||||
want = {
|
||||
"gate_w": ((EXPERTS, HIDDEN), torch.bfloat16),
|
||||
"bias": ((EXPERTS,), torch.float32),
|
||||
"w13": ((EXPERTS, 2 * d.inter, HIDDEN // 2), fp4),
|
||||
"w13_s": ((EXPERTS * 2 * d.inter, HIDDEN // 32), u8),
|
||||
"w2": ((EXPERTS, HIDDEN, d.inter // 2), fp4),
|
||||
"w2_s": ((EXPERTS * HIDDEN, d.down_scale_cols), u8),
|
||||
"sgu": ((2 * d.sh_inter, HIDDEN), e4m3),
|
||||
"sgu_s": ((2 * d.sh_inter // 32, HIDDEN // 32), u8),
|
||||
"sw2": ((HIDDEN, d.sh_inter), e4m3),
|
||||
"sw2_s": ((HIDDEN // 32, d.sh_inter // 32), u8),
|
||||
}
|
||||
_check_tensors(self, "MoE", want)
|
||||
|
||||
|
||||
class DSV41MonoLayer:
|
||||
"""The mono decode runner of one TP rank (``group``: its TP group, for the
|
||||
peer-memory handle exchange; a gloo / CPU group)."""
|
||||
|
||||
def __init__(self, tp: int, rank: int, group, device: torch.device | str = "cuda"):
|
||||
_ensure_writable_flydsl_cache()
|
||||
from .common.peer_memory import PeerBuffer
|
||||
|
||||
self.tp, self.rank = tp, rank
|
||||
self.d = Dims(tp)
|
||||
dev = self.device = torch.device(device)
|
||||
self._scratch: dict[int, torch.Tensor] = {}
|
||||
# [epoch, -, -, -, a mark per CTA, the MoE's counters]
|
||||
self.epoch = torch.zeros(EPOCH_WORDS, dtype=torch.int32, device=dev)
|
||||
self.q = torch.zeros(
|
||||
MAX_TOKENS, self.d.heads, HEAD_DIM, dtype=torch.bfloat16, device=dev
|
||||
)
|
||||
self.kt = torch.zeros(MAX_TOKENS * KEYS, dtype=torch.int32, device=dev)
|
||||
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
|
||||
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
|
||||
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
|
||||
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
|
||||
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self._first_pre[:, 0] = 1.0
|
||||
# an index source's K1 outputs, vLLM's indexer's inputs
|
||||
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
|
||||
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
|
||||
# the attention seam's outputs, K1 -> K2
|
||||
self.res_mid = torch.zeros(
|
||||
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
|
||||
)
|
||||
self.post_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self.comb_a = torch.zeros(MAX_TOKENS, HC, HC, dtype=torch.float32, device=dev)
|
||||
self.pre_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self.peer = PeerBuffer(2 * peer_half_bytes(tp), group, rank, tp, dev)
|
||||
self.peer.bytes.zero_()
|
||||
self._qk_scale = (
|
||||
torch.tensor([HEAD_DIM**-0.5 * _LOG2E], dtype=torch.float32)
|
||||
.view(torch.int32)
|
||||
.item()
|
||||
)
|
||||
self._kernels: dict = {}
|
||||
|
||||
def scratch(self, tokens: int) -> torch.Tensor:
|
||||
"""Step width ``tokens``'s scratch: a width's layout has memory of its own
|
||||
(``layer.scratch_layout``). Allocated at the width's first step, which is
|
||||
eager: vLLM runs every graph's batch eagerly before capturing it."""
|
||||
buf = self._scratch.get(tokens)
|
||||
if buf is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
f"DSv4.1 mono decode: step width {tokens} first reached inside "
|
||||
"a CUDA graph capture"
|
||||
)
|
||||
buf = torch.zeros(
|
||||
scratch_bytes(tokens, self.tp), dtype=torch.uint8, device=self.device
|
||||
)
|
||||
self._scratch[tokens] = buf
|
||||
return buf
|
||||
|
||||
def first_pre_mix(self, tokens: int) -> torch.Tensor:
|
||||
return self._first_pre[:tokens]
|
||||
|
||||
@staticmethod
|
||||
def supports(tokens: int) -> bool:
|
||||
return 1 <= tokens <= MAX_TOKENS
|
||||
|
||||
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
|
||||
key = (tokens, ratio, fold, index)
|
||||
if key not in self._kernels:
|
||||
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
|
||||
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
|
||||
return self._kernels[key]
|
||||
|
||||
def ffn_kernel(self, tokens: int):
|
||||
key = (tokens, "ffn")
|
||||
if key not in self._kernels:
|
||||
self._kernels[key] = build_mono_ffn(MonoBuild(tokens, self.tp, 0))
|
||||
return self._kernels[key]
|
||||
|
||||
@staticmethod
|
||||
def _outs(M: int, residual: torch.Tensor) -> tuple:
|
||||
dev = residual.device
|
||||
return (
|
||||
torch.empty(M, HIDDEN, dtype=torch.bfloat16, device=dev),
|
||||
torch.empty_like(residual),
|
||||
torch.empty(M, HC, 1, dtype=torch.float32, device=dev),
|
||||
torch.empty(M, HC, HC, dtype=torch.float32, device=dev),
|
||||
torch.empty(M, HC, dtype=torch.float32, device=dev),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
w: MonoLayerWeights,
|
||||
x: torch.Tensor, # [M, 5120] bf16: the previous FFN output (reduced)
|
||||
residual: torch.Tensor, # [M, 4, 5120] bf16
|
||||
post_mix: torch.Tensor, # [M, 4, 1] f32
|
||||
res_mix: torch.Tensor, # [M, 4, 4] f32
|
||||
pre_mix: torch.Tensor, # [M, 4] f32
|
||||
positions: torch.Tensor, # [M] int64
|
||||
slot_mapping: torch.Tensor, # [M] int64
|
||||
swa_cache: torch.Tensor, # [blocks, block, 584] uint8
|
||||
swa_indices: torch.Tensor, # [>= M, 1, 128] int32
|
||||
swa_lens: torch.Tensor, # [>= M] int32
|
||||
token_to_req: torch.Tensor, # [>= M] int32
|
||||
topk_indices: torch.Tensor | None = None,
|
||||
comp_cache: torch.Tensor | None = None,
|
||||
comp_block_table: torch.Tensor | None = None,
|
||||
outs: tuple | None = None,
|
||||
fold: bool = True,
|
||||
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
|
||||
):
|
||||
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
|
||||
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
|
||||
source's top-k selection into ``topk_indices``, run between the
|
||||
kernels on K1's normed input and qr."""
|
||||
M = x.shape[0]
|
||||
assert self.supports(M), M
|
||||
a = w.attn
|
||||
assert a is not None
|
||||
ratio = a.ratio
|
||||
index = select is not None
|
||||
assert not index or ratio
|
||||
k1, k2 = self.kernels(M, ratio, fold, index)
|
||||
st = torch.cuda.current_stream()
|
||||
if outs is None:
|
||||
outs = self._outs(M, residual)
|
||||
out, res_out, post_out, comb_out, pre_out = outs
|
||||
for t_ in (x, residual, post_mix, res_mix, pre_mix, positions, slot_mapping):
|
||||
assert t_.is_contiguous()
|
||||
assert positions.dtype == slot_mapping.dtype == torch.int64
|
||||
assert swa_indices.dtype == swa_lens.dtype == token_to_req.dtype == torch.int32
|
||||
assert swa_indices.stride(0) == 128 and swa_cache.dtype == torch.uint8
|
||||
assert swa_cache.shape[-1] == 584 and swa_cache.stride(1) == 584
|
||||
swa_block = swa_cache.shape[1]
|
||||
if ratio:
|
||||
assert topk_indices is not None and comp_cache is not None
|
||||
assert comp_block_table is not None
|
||||
assert topk_indices.dtype == torch.int32 and topk_indices.stride(0) == 512
|
||||
assert comp_cache.dtype == torch.uint8 and comp_cache.stride(1) == 584
|
||||
assert comp_block_table.dtype == torch.int32
|
||||
comp, comp_stride, comp_block = (
|
||||
comp_cache,
|
||||
comp_cache.stride(0),
|
||||
comp_cache.shape[1],
|
||||
)
|
||||
bt, bt_stride, topk = (
|
||||
comp_block_table,
|
||||
comp_block_table.stride(0),
|
||||
topk_indices,
|
||||
)
|
||||
else:
|
||||
comp, comp_stride, comp_block = self._dummy, 0, 1
|
||||
bt, bt_stride, topk = self._dummy, 0, self._dummy
|
||||
k1(
|
||||
residual.data_ptr(),
|
||||
x.data_ptr(),
|
||||
post_mix.data_ptr(),
|
||||
res_mix.data_ptr(),
|
||||
pre_mix.data_ptr(),
|
||||
w.hc_attn_fn.data_ptr(),
|
||||
w.hc_attn_scale.data_ptr(),
|
||||
w.hc_attn_base.data_ptr(),
|
||||
w.attn_norm.data_ptr(),
|
||||
self.res_mid.data_ptr(),
|
||||
self.post_a.data_ptr(),
|
||||
self.comb_a.data_ptr(),
|
||||
self.pre_a.data_ptr(),
|
||||
a.wqkv.data_ptr(),
|
||||
a.wqkv_scale.data_ptr(),
|
||||
a.q_norm.data_ptr(),
|
||||
a.kv_norm.data_ptr(),
|
||||
a.wq_b.data_ptr(),
|
||||
a.wq_b_scale.data_ptr(),
|
||||
a.cos_sin.data_ptr(),
|
||||
positions.data_ptr(),
|
||||
slot_mapping.data_ptr(),
|
||||
swa_cache.data_ptr(),
|
||||
swa_cache.stride(0),
|
||||
swa_block,
|
||||
self.q.data_ptr(),
|
||||
swa_indices.data_ptr(),
|
||||
swa_lens.data_ptr(),
|
||||
comp_block,
|
||||
bt.data_ptr(),
|
||||
bt_stride,
|
||||
topk.data_ptr(),
|
||||
token_to_req.data_ptr(),
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
self.normed.data_ptr(),
|
||||
self.qr.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.epoch.data_ptr(),
|
||||
0,
|
||||
stream=st,
|
||||
)
|
||||
if select is not None:
|
||||
select(self.normed[:M], self.qr[:M])
|
||||
k2(
|
||||
self.q.data_ptr(),
|
||||
swa_cache.data_ptr(),
|
||||
swa_cache.stride(0),
|
||||
swa_block,
|
||||
comp.data_ptr(),
|
||||
comp_stride,
|
||||
comp_block,
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
slot_mapping.data_ptr(),
|
||||
swa_indices.data_ptr(),
|
||||
swa_lens.data_ptr(),
|
||||
token_to_req.data_ptr(),
|
||||
topk.data_ptr(),
|
||||
bt.data_ptr(),
|
||||
bt_stride,
|
||||
positions.data_ptr(),
|
||||
a.attn_sink.data_ptr(),
|
||||
self._qk_scale,
|
||||
a.cos_sin.data_ptr(),
|
||||
a.wo_a.data_ptr(),
|
||||
a.wo_a_scale.data_ptr(),
|
||||
a.wo_b.data_ptr(),
|
||||
a.wo_b_scale.data_ptr(),
|
||||
self._zero_rec.data_ptr(),
|
||||
self.res_mid.data_ptr(),
|
||||
self.post_a.data_ptr(),
|
||||
self.comb_a.data_ptr(),
|
||||
self.pre_a.data_ptr(),
|
||||
w.hc_ffn_fn.data_ptr(),
|
||||
w.hc_ffn_scale.data_ptr(),
|
||||
w.hc_ffn_base.data_ptr(),
|
||||
w.ffn_norm.data_ptr(),
|
||||
res_out.data_ptr(),
|
||||
post_out.data_ptr(),
|
||||
comb_out.data_ptr(),
|
||||
pre_out.data_ptr(),
|
||||
w.gate_w.data_ptr(),
|
||||
w.bias.data_ptr(),
|
||||
w.w13.data_ptr(),
|
||||
w.w13_s.data_ptr(),
|
||||
w.w2.data_ptr(),
|
||||
w.w2_s.data_ptr(),
|
||||
w.sgu.data_ptr(),
|
||||
w.sgu_s.data_ptr(),
|
||||
w.sw2.data_ptr(),
|
||||
w.sw2_s.data_ptr(),
|
||||
out.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.peer.local,
|
||||
self.peer.addresses.data_ptr(),
|
||||
self.rank,
|
||||
self.epoch.data_ptr(),
|
||||
0,
|
||||
stream=st,
|
||||
)
|
||||
return outs
|
||||
|
||||
def ffn(
|
||||
self,
|
||||
w: MonoLayerWeights,
|
||||
part: torch.Tensor, # [M, 5120] bf16: this rank's unreduced wo_b output
|
||||
residual: torch.Tensor, # [M, 4, 5120] bf16, after the attention seam
|
||||
post_mix: torch.Tensor, # [M, 4, 1] f32, the attention seam's
|
||||
res_mix: torch.Tensor, # [M, 4, 4] f32
|
||||
pre_mix: torch.Tensor, # [M, 4] f32
|
||||
outs: tuple | None = None,
|
||||
):
|
||||
"""-> (out, residual, post_mix, res_mix, pre_mix) of a layer whose
|
||||
attention vLLM ran: the attention's TP reduce, the FFN seam, the MoE and
|
||||
its all-reduce in one launch."""
|
||||
M = part.shape[0]
|
||||
assert self.supports(M), M
|
||||
for t_ in (part, residual, post_mix, res_mix, pre_mix):
|
||||
assert t_.is_contiguous()
|
||||
assert part.dtype == residual.dtype == torch.bfloat16
|
||||
if outs is None:
|
||||
outs = self._outs(M, residual)
|
||||
out, res_out, post_out, comb_out, pre_out = outs
|
||||
self.ffn_kernel(M)(
|
||||
part.data_ptr(),
|
||||
residual.data_ptr(),
|
||||
post_mix.data_ptr(),
|
||||
res_mix.data_ptr(),
|
||||
pre_mix.data_ptr(),
|
||||
w.hc_ffn_fn.data_ptr(),
|
||||
w.hc_ffn_scale.data_ptr(),
|
||||
w.hc_ffn_base.data_ptr(),
|
||||
w.ffn_norm.data_ptr(),
|
||||
res_out.data_ptr(),
|
||||
post_out.data_ptr(),
|
||||
comb_out.data_ptr(),
|
||||
pre_out.data_ptr(),
|
||||
w.gate_w.data_ptr(),
|
||||
w.bias.data_ptr(),
|
||||
w.w13.data_ptr(),
|
||||
w.w13_s.data_ptr(),
|
||||
w.w2.data_ptr(),
|
||||
w.w2_s.data_ptr(),
|
||||
w.sgu.data_ptr(),
|
||||
w.sgu_s.data_ptr(),
|
||||
w.sw2.data_ptr(),
|
||||
w.sw2_s.data_ptr(),
|
||||
out.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.peer.local,
|
||||
self.peer.addresses.data_ptr(),
|
||||
self.rank,
|
||||
self.epoch.data_ptr(),
|
||||
stream=torch.cuda.current_stream(),
|
||||
)
|
||||
return outs
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The mono layer's kernel sources, in every build's JIT cache key
|
||||
(``common.plan.key_tuple``): an edit anywhere in this package rebuilds."""
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import source_digest
|
||||
|
||||
SOURCES = source_digest(".")
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The V4.1 stages the mono layer adapts from ROCm/ATOM: the mHC seams
|
||||
(``seam``), the MoE (``moe``) and the GEMV / LDS helpers they share (``gemv``)."""
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/dims.py
|
||||
"""A TP rank's share of V4.1's MoE widths: every constant the MoE stages derive
|
||||
from the TP size, in one place (a build key's ``tp`` picks it)."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import Shard, padded
|
||||
|
||||
# the model's full widths
|
||||
MOE_INTER = 2304 # a routed expert's intermediate width
|
||||
SHARED_INTER = 2304 # the shared expert's
|
||||
# the loader pads a routed expert's rank share to a multiple of this (``FusedMoE``
|
||||
# ``pad_align``) with zero weights
|
||||
MOE_PAD = 128
|
||||
ROWS = 16 # GEMV rows a task
|
||||
UG_PART = 32 # intermediate columns an ug task part: one FP4 group
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dims:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
shard = Shard(self.tp)
|
||||
shard.split(MOE_INTER, "routed intermediate")
|
||||
shard.split(SHARED_INTER, "shared intermediate")
|
||||
|
||||
@property
|
||||
def inter_real(self) -> int:
|
||||
"""A routed expert's real intermediate width on this rank."""
|
||||
return MOE_INTER // self.tp
|
||||
|
||||
@property
|
||||
def inter(self) -> int:
|
||||
"""... padded as the loader pads it (the rest zero weights)."""
|
||||
return padded(self.inter_real, MOE_PAD)
|
||||
|
||||
@property
|
||||
def sh_inter(self) -> int:
|
||||
"""The shared expert's intermediate width on this rank."""
|
||||
return SHARED_INTER // self.tp
|
||||
|
||||
@property
|
||||
def shared_tasks(self) -> int:
|
||||
return self.sh_inter // ROWS
|
||||
|
||||
@property
|
||||
def mid_words(self) -> int:
|
||||
"""MXFP8 words of a pick's routed intermediate."""
|
||||
return self.inter // 4
|
||||
|
||||
@property
|
||||
def down_scale_cols(self) -> int:
|
||||
"""w2_s's e8m0 columns: inter / 32, padded to a multiple of 8
|
||||
(aiter ``shuffle_scale``)."""
|
||||
return padded(self.inter // 32, 8)
|
||||
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (its GEMV and LDS helpers).
|
||||
"""The FP8 GEMV tiles of the seam and MoE stages and the helpers they share:
|
||||
token tiles, LDS rows of MXFP8 words, scalar loads and the mailbox-to-LDS
|
||||
copy."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.mx import (
|
||||
UNIT_SCALE,
|
||||
mfma_scaled,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
CM_NT,
|
||||
fresh,
|
||||
rsrc,
|
||||
traced,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
|
||||
from vllm.models.deepseek_v41.amd.mono.common.sync import POLL_MAX
|
||||
|
||||
ROWS = 16 # GEMV rows a task
|
||||
TILE = 16 # an MFMA's N: the tokens a GEMV pass takes (``token_tiles``)
|
||||
|
||||
|
||||
def ld_f32(ptr, i):
|
||||
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
|
||||
|
||||
|
||||
def ld_bf(ptr, i):
|
||||
return fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
)
|
||||
|
||||
|
||||
def token_tiles(s):
|
||||
"""A step's tokens in GEMV passes: [(first token, tokens)]. A GEMV loads its
|
||||
weights once and runs every pass on them, an x tile in LDS at a time."""
|
||||
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
|
||||
|
||||
|
||||
def tile_rows(s):
|
||||
"""The LDS rows an x tile holds."""
|
||||
return min(s, TILE)
|
||||
|
||||
|
||||
# a token's LDS row of MXFP8 words / E8M0 codes, 16 of which a MFMA B operand
|
||||
# gathers, is padded by 16 B: at a multiple of 64 words they share a bank group
|
||||
LDS_PAD = 4
|
||||
|
||||
|
||||
def lds_row(words):
|
||||
return words + LDS_PAD
|
||||
|
||||
|
||||
def lds_at(u, row):
|
||||
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
|
||||
return u + u // row * LDS_PAD
|
||||
|
||||
|
||||
def lds_tail(k):
|
||||
"""Words a K = ``k`` row's last K step reads past its ``lds_row`` (selected
|
||||
out): the last row's buffer must hold them."""
|
||||
return max(0, 32 * -(-k // 128) - lds_row(k // 4))
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
"""Token ``v`` of the tile from ``t0`` (a first tile adds nothing; ``t0``
|
||||
traced: a tile a CTA picks at run time)."""
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- GEMV
|
||||
def _fp8_step_load(c, r_w, r_ws, k, rg, st, row_major=False):
|
||||
"""K step st's global operands of rows 16 rg ..: the two 64-column weight
|
||||
chunks (the upper clamped into the row) and the raw scale word. The (16, 16)
|
||||
shuffle lays a row group out in 32-column blocks of 16 rows, a chunk two of
|
||||
them (lanes 0-31, 32-63); ``k`` % 64 = 32 (a last quarter step): the lanes
|
||||
of a block past the row read its last block. ``row_major`` (vLLM's dense
|
||||
layout, k % 64 == 0): the same registers from a row-major [N, K] weight --
|
||||
lane l row 16 rg + l % 16, K bytes 128 st + 16 (l // 16) .. and + 64."""
|
||||
lane = c["lane"]
|
||||
j = lane // 16
|
||||
kg = k // 32
|
||||
if const_expr(row_major):
|
||||
assert k % 64 == 0
|
||||
row = rg * 16 + lane % 16
|
||||
lo_at = row * (k // 4) + st * 32 + j * 4
|
||||
hi_at = row * (k // 4) + fx.min(2 * st + 1, k // 64 - 1) * 16 + j * 4
|
||||
lo = fx.Vector(
|
||||
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
hi = fx.Vector(
|
||||
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
if const_expr(k % 64 == 0):
|
||||
lo_at = (rg * (k // 64) + 2 * st) * 256 + lane * 4
|
||||
else:
|
||||
lo_at = rg * (k * 4) + fx.min(4 * st + lane // 32, kg - 1) * 128 + lane % 32 * 4
|
||||
lo = fx.Vector(
|
||||
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
if const_expr(k % 64 == 0):
|
||||
hi_at = (rg * (k // 64) + fx.min(2 * st + 1, k // 64 - 1)) * 256 + lane * 4
|
||||
else:
|
||||
hi_at = rg * (k * 4) + fx.min(2 * st + 1, k // 64 - 1) * 256 + lane * 4
|
||||
hi = fx.Vector(
|
||||
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def _fp8_step_mfma(c, k, xl, xsl, st, ops, acc, live=None, rows=None):
|
||||
"""``acc`` through K step st's scaled MFMA: ``_fp8_step_load``'s operands
|
||||
against every token's MXFP8 row in LDS. ``live`` false: a step past the end
|
||||
(its operands those of the last step), zero at unit scale. ``k`` % 64 = 32:
|
||||
a lane's lower block past the row is zero on both sides."""
|
||||
s, lane = c["S"] if rows is None else rows, c["lane"]
|
||||
lo, hi, sa = ops
|
||||
kg = k // 32
|
||||
x_row, s_row = lds_row(k // 4), lds_row(kg)
|
||||
j = lane // 16
|
||||
col = fx.min(lane % 16, s - 1)
|
||||
if const_expr(live is not None):
|
||||
st = fx.min(st, (k + 127) // 128 - 1)
|
||||
lo = [live.select(lo[d], fx.Int32(0)) for d in range(4)]
|
||||
past = (2 * st + 1) * 64 >= k
|
||||
if const_expr(live is not None):
|
||||
past = past | ~live
|
||||
av = fx.Vector.from_elements(
|
||||
[lo[d] for d in range(4)] + [past.select(fx.Int32(0), hi[d]) for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
grp = st * 4 + j
|
||||
xlo = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xhi = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + 16 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
if const_expr(k % 64 != 0):
|
||||
over = 4 * st + j // 2 >= kg
|
||||
av = fx.Vector.from_elements(
|
||||
[over.select(fx.Int32(0), av[d]) for d in range(4)]
|
||||
+ [av[4 + d] for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
xlo = [over.select(fx.Int32(0), xlo[d]) for d in range(4)]
|
||||
xv = fx.Vector.from_elements(
|
||||
[xlo[d] for d in range(4)]
|
||||
+ [past.select(fx.Int32(0), xhi[d]) for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
sb = fx.ptr_load(xsl + (col * s_row + fx.min(grp, kg - 1)))
|
||||
unit = grp >= kg
|
||||
if const_expr(live is not None):
|
||||
unit = unit | ~live
|
||||
sa = unit.select(fx.Int32(UNIT_SCALE), sa)
|
||||
sb = unit.select(fx.Int32(UNIT_SCALE), sb)
|
||||
return mfma_scaled(av, xv, acc, sa, sb)
|
||||
|
||||
|
||||
def _gemv_steps(c, k, split, tiled=False):
|
||||
"""This wave's K steps (a step past the end marked, ``split`` waves past
|
||||
split repeat the last one's; ``tiled``: wave w takes part w % split of tile
|
||||
w / split): [(step, live)]."""
|
||||
wave = c["wave"]
|
||||
steps = (k + 127) // 128
|
||||
if const_expr(split is None):
|
||||
per = (steps + WAVES - 1) // WAVES
|
||||
ragged = steps % WAVES != 0
|
||||
return [
|
||||
(wave + WAVES * i, (wave + WAVES * i < steps) if ragged else None)
|
||||
for i in range(per)
|
||||
]
|
||||
assert split <= WAVES and steps % split == 0
|
||||
per = steps // split
|
||||
part = wave % split if tiled else fx.min(wave, split - 1)
|
||||
return [(part * per + i, None) for i in range(per)]
|
||||
|
||||
|
||||
def gemv_fp8_loads(c, w, ws, k, rg, split=None, tiled=False, row_major=False):
|
||||
"""The weight operands of ``gemv_fp8``'s rows for this wave, every one in
|
||||
flight at once (a step's latency each would put them in series): loads
|
||||
only. A step past the end loads the last step. ``tiled`` (split x tiles =
|
||||
WAVES): row groups rg .., one a group of ``split`` waves."""
|
||||
steps = (k + 127) // 128
|
||||
r_w, r_ws = rsrc(w), rsrc(ws)
|
||||
if const_expr(tiled):
|
||||
assert WAVES % split == 0
|
||||
rg = rg + c["wave"] // split
|
||||
return [
|
||||
_fp8_step_load(
|
||||
c,
|
||||
r_w,
|
||||
r_ws,
|
||||
k,
|
||||
rg,
|
||||
st if live is None else fx.min(st, steps - 1),
|
||||
row_major,
|
||||
)
|
||||
for st, live in _gemv_steps(c, k, split, tiled)
|
||||
]
|
||||
|
||||
|
||||
def gemv_fp8_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=None):
|
||||
"""``gemv_fp8_loads``' operands against the rows in LDS (``rows``: a token
|
||||
tile's, every token's by default) ->
|
||||
``red[wave][lane]``; the scaled 16x16x128 MFMA, chunk kc in dwords 0-3 and kc +
|
||||
1 in 4-7. ``k`` a multiple of 32: a last half step has its upper chunk zeroed
|
||||
on both sides at unit scale (a NaN byte past the row must not reach the
|
||||
MFMA), its loads clamped into the row: the last row group's would read past
|
||||
the tensor; a last quarter step its lower chunk's upper block too. A step
|
||||
past the end adds zero at unit scale.
|
||||
|
||||
The K order is the original kernel's, which ``row_sum`` completes:
|
||||
wave w takes steps w, w + 8, ... (``split`` None: aiter's preshuffled group32
|
||||
GEMM, P0.5 microbenchmark 1), or waves 0 .. split - 1 one contiguous K range
|
||||
each, the other waves zero (aiter's split-K bmm, whose last split sums the
|
||||
partials in split order)."""
|
||||
lane, wave = c["lane"], c["wave"]
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i, (st, live) in enumerate(_gemv_steps(c, k, split, tiled)):
|
||||
acc = _fp8_step_mfma(c, k, xl, xsl, st, ops[i], acc, live, rows)
|
||||
if const_expr(split is not None and not tiled):
|
||||
acc = fx.Vector.from_elements(
|
||||
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
|
||||
fx.Float32,
|
||||
)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- mailbox -> LDS
|
||||
@traced
|
||||
def poll_copy(c, idx, width, spans, live=None):
|
||||
"""Mailbox spans [(region, pair_of, n, dst, row)] into LDS: element u (pair
|
||||
``pair_of(u)``) to dst + u, or ``lds_at(u, row)`` for a span of rows
|
||||
``row`` words wide. Thread ``idx`` of ``width`` takes u = idx, idx +
|
||||
width, ..., POLL_MAX a round trip, each batch stored before the next is
|
||||
polled (holding them all spills past S = 16); a thread past a span's end
|
||||
re-reads its last element and stores nothing.
|
||||
|
||||
``live``: a span's elements the reader needs (traced, <= n), a span each;
|
||||
a batch past them is not polled and an element past them neither polled
|
||||
nor written (one past a step's rows is never written)."""
|
||||
idx = fresh(idx) # its elements' indices not hoisted out of a task loop
|
||||
if const_expr(live is None):
|
||||
_poll_store_batches(
|
||||
c,
|
||||
[
|
||||
(region, pair_of, n, (dst, row), idx + width * i)
|
||||
for region, pair_of, n, dst, row in spans
|
||||
for i in range((n + width - 1) // width)
|
||||
],
|
||||
)
|
||||
return
|
||||
for (region, pair_of, n, dst, row), need in zip(spans, live):
|
||||
chunks = [
|
||||
(region, pair_of, need, (dst, row), idx + width * i)
|
||||
for i in range((n + width - 1) // width)
|
||||
]
|
||||
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
|
||||
if chunks[b0][4] < need:
|
||||
_poll_store_batches(c, chunks[b0 : b0 + POLL_MAX])
|
||||
|
||||
|
||||
@traced
|
||||
def _poll_store_batches(c, chunks):
|
||||
"""``poll_copy``'s chunks [(region, pair_of, n, (dst, row), u)], POLL_MAX a
|
||||
round trip, each batch stored before the next is polled."""
|
||||
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
|
||||
batch = chunks[b0 : b0 + POLL_MAX]
|
||||
got = c["poll"]([(r, p(fx.min(u, n - 1)), 1) for r, p, n, _, u in batch])
|
||||
for (_, _, n, (dst, row), u), v in zip(batch, got):
|
||||
at = u if row is None else lds_at(u, row)
|
||||
if u < n:
|
||||
fx.ptr_store(v[0], dst + at)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/moe_plan.py
|
||||
"""Every per-step decision of which MoE ug flags (UGF) and intermediate rows
|
||||
(MID) the ug stage writes and the down stage reads, defined once.
|
||||
|
||||
The route table (``stage_route``) gives each expert with picks a slot of U, its
|
||||
picks a run from ``slot_first``, and, past one token tile, its picks tiles of
|
||||
``TILE`` from ``tile_first`` (each tile's slot ``tile_slot``). An ug task covers
|
||||
a unit -- a slot, or past one token tile a pick tile -- and one of ``ug_parts`` parts
|
||||
of the intermediate; its flag is ``ug_flag``. A down round polls each of its
|
||||
slots' flags (``polled_tile`` its tiles) before it reads their MID rows. Called
|
||||
by the kernel on traced Int32 and by the CPU property test on Python ints.
|
||||
"""
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import cdiv, imin
|
||||
|
||||
TILE = 16 # picks a pick tile: an MFMA's N
|
||||
|
||||
|
||||
def pick_tiles(count):
|
||||
"""A slot's pick tiles: its ``count`` picks, ``TILE`` a tile."""
|
||||
return cdiv(count, TILE)
|
||||
|
||||
|
||||
def ug_parts(groups, ug_groups):
|
||||
"""A unit's ug parts: the expert's ``groups`` FP4 groups of its real width,
|
||||
``ug_groups`` a part. A part wholly in the padding is not run: MID's padded
|
||||
columns stay the step's zeros, against zero weights in down."""
|
||||
return cdiv(groups, ug_groups)
|
||||
|
||||
|
||||
def ug_flag(unit, part, ug_parts):
|
||||
"""The UGF pair of ug task (``unit``, ``part``), ``ug_parts`` parts a unit."""
|
||||
return unit * ug_parts + part
|
||||
|
||||
|
||||
def ug_tasks(units, ug_parts):
|
||||
"""A step's ug tasks: every part of each of its ``units``."""
|
||||
return units * ug_parts
|
||||
|
||||
|
||||
def tile_pick0(slot_first, tile, tile_first):
|
||||
"""Pick tile ``tile``'s first pick, its slot's picks from ``slot_first`` and
|
||||
tiles from ``tile_first``."""
|
||||
return slot_first + (tile - tile_first) * TILE
|
||||
|
||||
|
||||
def polled_tile(tile_first, tiles, j):
|
||||
"""The pick tile a down round polls as a slot's ``j``-th, the slot's
|
||||
``tiles`` tiles from ``tile_first``: the last past them."""
|
||||
return tile_first + imin(j, tiles - 1)
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/moe_shape.py
|
||||
"""The MoE stages' build key and route shape: what they are built for (``MoeBuild``),
|
||||
and the task counts, placements and LDS route table that follow from its
|
||||
routing (``RouteShape``), with the table's readers. Plain Python but for the
|
||||
readers; the kernel's stages are in ``moe``."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import flydsl.expr as fx
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import BLOCKS, WAVES
|
||||
from vllm.models.deepseek_v41.amd.mono.stages import moe_plan as mplan
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.dims import UG_PART, Dims
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.gemv import ROWS, TILE, token_tiles
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.router import PARTS as ROUTER_PARTS
|
||||
|
||||
EXPERTS, TOPK = 384, 6 # the target's routing (a build's: ``MoeBuild``)
|
||||
ROUTER0 = 8 # the router's first task: past the xq tasks (a token each)
|
||||
MASK_WORDS = 2 # an expert's token bitmap in the route table: tokens <= 64
|
||||
# a route lane's experts (EXPERTS / 64) sorted descending, ties kept in index
|
||||
# order: the network by their count
|
||||
SORT_NETS = {
|
||||
6: [
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(4, 5),
|
||||
(0, 2),
|
||||
(1, 4),
|
||||
(3, 5),
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(4, 5),
|
||||
(1, 2),
|
||||
(3, 4),
|
||||
(2, 3),
|
||||
],
|
||||
2: [(0, 1)],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoeBuild:
|
||||
tokens: int
|
||||
# routed experts and a token's top-k: the target's, or the DSpark draft's
|
||||
experts: int = EXPERTS
|
||||
topk: int = TOPK
|
||||
# the TP size: every width a rank holds (``Dims``)
|
||||
tp: int = 4
|
||||
timeline: bool = field(default=False, metadata={"sym": "tl"})
|
||||
# FP4 groups an ug task (``RouteShape.ug_groups``)
|
||||
ug_groups: int = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteShape:
|
||||
"""A build's routing: ``experts`` routed experts, ``topk`` a token of
|
||||
``tokens``, the task counts and placements that follow, and the route table
|
||||
``load_route`` lays out in LDS: every token's (ids, weights), |U|, the
|
||||
experts of U, each (t, k)'s slot in U, each expert's slot, the picks in slot
|
||||
order and each slot's first position in it (past one token tile also each
|
||||
slot's first pick tile and each pick tile's slot)."""
|
||||
|
||||
experts: int
|
||||
topk: int
|
||||
tokens: int
|
||||
tp: int
|
||||
# FP4 groups (32 intermediate columns) an ug task, its waves splitting K in
|
||||
# halves (2) or quarters (1). 1 is twice the tasks, so the CTAs' ug ends
|
||||
# closer together: faster at the draft's widths (40 rows -4 us), slower at
|
||||
# the target's (48 rows +3 us; the runners pick)
|
||||
ug_groups: int = 2
|
||||
|
||||
@property
|
||||
def dims(self) -> Dims:
|
||||
return Dims(self.tp)
|
||||
|
||||
@property
|
||||
def ug_parts(self) -> int:
|
||||
"""An expert's ug tasks (``moe_plan.ug_parts``)."""
|
||||
return mplan.ug_parts(self.dims.inter_real // UG_PART, self.ug_groups)
|
||||
|
||||
@property
|
||||
def ug_k_slices(self) -> int:
|
||||
"""The K slices of an ug task's waves: a wave a (group, 16-column half,
|
||||
slice)."""
|
||||
return WAVES // (2 * self.ug_groups)
|
||||
|
||||
@property
|
||||
def picks(self) -> int:
|
||||
return self.tokens * self.topk
|
||||
|
||||
@property
|
||||
def max_u(self) -> int:
|
||||
return min(self.picks, self.experts)
|
||||
|
||||
@property
|
||||
def router_groups(self) -> tuple[int, int]:
|
||||
"""``stage_router``'s token groups and their tokens: a group a token
|
||||
tile, as many as one round of CTAs holds (a CTA running two serialized
|
||||
them: the route waited on the 32 that did at 48 rows)."""
|
||||
per_group = self.experts // ROWS * ROUTER_PARTS
|
||||
groups = max(1, min(len(token_tiles(self.tokens)), BLOCKS // per_group))
|
||||
tokens = -(-self.tokens // groups)
|
||||
assert (groups - 1) * tokens < self.tokens
|
||||
return groups, tokens
|
||||
|
||||
@property
|
||||
def router_tasks(self) -> int:
|
||||
# (token group, 16-row tile, K part): ``stage_router``
|
||||
return self.router_groups[0] * self.experts // ROWS * ROUTER_PARTS
|
||||
|
||||
@property
|
||||
def router0(self) -> int:
|
||||
"""Past the xq tasks (a token each from CTA 0)."""
|
||||
return max(ROUTER0, self.tokens)
|
||||
|
||||
@property
|
||||
def route0(self) -> int:
|
||||
return self.router0 + self.router_tasks
|
||||
|
||||
@property
|
||||
def shared0(self) -> int:
|
||||
return self.route0 + max(8, self.tokens)
|
||||
|
||||
@property
|
||||
def shared_units(self) -> int:
|
||||
"""The shared expert's (task, token tile) units (``stage_shared``)."""
|
||||
return self.dims.shared_tasks * len(token_tiles(self.tokens))
|
||||
|
||||
@property
|
||||
def ug0(self) -> int:
|
||||
"""The first ug round's placement: the shared units' CTAs last."""
|
||||
return self.shared0 + self.shared_units
|
||||
|
||||
@property
|
||||
def sort_net(self):
|
||||
return SORT_NETS[self.experts // 64]
|
||||
|
||||
@property
|
||||
def tab_words(self) -> int:
|
||||
return self.tile_slot0 + self.max_tiles if self.multi else self.queue0 + 1
|
||||
|
||||
@property
|
||||
def multi(self) -> bool:
|
||||
"""More than one token tile: an ug task is a (tile of a slot's picks,
|
||||
part) (``stage_ug``)."""
|
||||
return self.tokens > TILE
|
||||
|
||||
@property
|
||||
def slot_tiles(self) -> int:
|
||||
"""A slot's pick tiles at most (a token picks an expert once)."""
|
||||
return len(token_tiles(self.tokens))
|
||||
|
||||
@property
|
||||
def max_tiles(self) -> int:
|
||||
"""The pick tiles of U at most: a slot's partial tile each, then the
|
||||
whole tiles, and never more than a tile a pick."""
|
||||
return min(self.picks, self.max_u + self.picks // TILE)
|
||||
|
||||
@property
|
||||
def tile_first0(self) -> int:
|
||||
"""Past one token tile: each slot's first pick tile, then their count."""
|
||||
return self.queue0 + 1
|
||||
|
||||
@property
|
||||
def tile_slot0(self) -> int:
|
||||
"""Past one token tile: each pick tile's slot of U."""
|
||||
return self.tile_first0 + self.max_u + 1
|
||||
|
||||
@property
|
||||
def queue0(self) -> int:
|
||||
"""The ug task this CTA took from the queue (``take_ug_task``)."""
|
||||
return self.mask0 + MASK_WORDS * self.experts
|
||||
|
||||
@property
|
||||
def expert_slot0(self) -> int:
|
||||
"""The route table's per-expert slots (a slot where the expert is in U)."""
|
||||
return 3 * self.picks + 1 + self.max_u
|
||||
|
||||
@property
|
||||
def order0(self) -> int:
|
||||
"""The picks (t topk + k) in slot order, a slot's ascending."""
|
||||
return self.expert_slot0 + self.experts
|
||||
|
||||
@property
|
||||
def first0(self) -> int:
|
||||
"""Each slot's first position in the pick order, then the pick count."""
|
||||
return self.order0 + self.picks
|
||||
|
||||
@property
|
||||
def mask0(self) -> int:
|
||||
"""Each expert's tokens that picked it, a bit a token (``MASK_WORDS``)."""
|
||||
return self.first0 + self.max_u + 1
|
||||
|
||||
@property
|
||||
def tile_picks(self) -> int:
|
||||
"""A slot's picks at most: a token picks an expert once."""
|
||||
return min(self.tokens, 16)
|
||||
|
||||
def route_w(self, tab, s, t, k):
|
||||
return fx.ptr_load(tab + (t * 2 * self.topk + self.topk + k)).bitcast(
|
||||
fx.Float32
|
||||
)
|
||||
|
||||
def n_union(self, tab, s):
|
||||
return fx.ptr_load(tab + 2 * s * self.topk)
|
||||
|
||||
def union_expert(self, tab, s, u):
|
||||
return fx.ptr_load(tab + (2 * s * self.topk + 1 + u))
|
||||
|
||||
def slot_first(self, tab, u):
|
||||
return fx.ptr_load(tab + (self.first0 + u))
|
||||
|
||||
def pick_at(self, tab, i):
|
||||
return fx.ptr_load(tab + (self.order0 + i))
|
||||
|
||||
def tile_first(self, tab, u):
|
||||
return fx.ptr_load(tab + (self.tile_first0 + u))
|
||||
|
||||
def tile_slot(self, tab, i):
|
||||
return fx.ptr_load(tab + (self.tile_slot0 + i))
|
||||
|
||||
|
||||
def route_shape(key: MoeBuild) -> RouteShape:
|
||||
return RouteShape(key.experts, key.topk, key.tokens, key.tp, key.ug_groups)
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/model_ops/deepseek_v41/router.py
|
||||
"""V4.1's router GEMV tile for decode: bf16 x [T, 5120] times the gate weight
|
||||
[384, 5120] -> f32 logits, in one fixed summation order.
|
||||
|
||||
Rows 16 a tile, K in ``PARTS`` contiguous parts, 16x16x32 bf16 MFMA: in part
|
||||
p wave w takes the part's K steps w, w + 8, ... (32 columns each) in that
|
||||
order, and the eight wave partials are summed in wave order
|
||||
(``common.ops.row_sum``); the MoE's route stage sums the parts in part
|
||||
order. The MoE stages spread a tile's parts over PARTS CTAs, and past 16 tokens
|
||||
a token tile a CTA too, which the order allows: a CTA alone cannot read a
|
||||
tile, or every token's x, fast enough.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import mfma_bf16, rsrc
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
|
||||
|
||||
HIDDEN = 5120
|
||||
ROWS = 16 # a tile's rows: the MFMA's M
|
||||
PARTS = 4 # contiguous K parts of a tile, summed in order
|
||||
|
||||
|
||||
def _k0s(part, lane, wave):
|
||||
"""The K offsets of this lane's loads in part ``part`` (8 bf16 each)."""
|
||||
per_wave = HIDDEN // 32 // WAVES // PARTS
|
||||
assert per_wave * 32 * WAVES * PARTS == HIDDEN
|
||||
return [
|
||||
(part * per_wave * WAVES + wave + WAVES * i) * 32 + 8 * (lane // 16)
|
||||
for i in range(per_wave)
|
||||
]
|
||||
|
||||
|
||||
def router_weight_loads(w, task, part, lane, wave, w_cm=0):
|
||||
"""``router_tile``'s weight operands (rows 16 task .., K part ``part``).
|
||||
``w_cm``: their load cache policy (nontemporal where nothing else of the
|
||||
step reuses them)."""
|
||||
row = task * ROWS + lane % 16
|
||||
return [
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(w),
|
||||
(row * HIDDEN + k0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=w_cm,
|
||||
)
|
||||
)
|
||||
for k0 in _k0s(part, lane, wave)
|
||||
]
|
||||
|
||||
|
||||
def router_x_loads(x, part, s, lane, wave, x_cm=0, t0=0):
|
||||
"""``router_tile``'s x operands: the s tokens from ``t0`` (column
|
||||
lane % 16; ``t0`` traced or an int). ``x_cm``: their load cache policy
|
||||
(device scope when another CTA of the same launch wrote x)."""
|
||||
tok = fx.min(lane % 16, s - 1)
|
||||
if const_expr(not isinstance(t0, int) or t0 != 0):
|
||||
tok = tok + t0
|
||||
return [
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(x),
|
||||
(tok * HIDDEN + k0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=x_cm,
|
||||
)
|
||||
)
|
||||
for k0 in _k0s(part, lane, wave)
|
||||
]
|
||||
|
||||
|
||||
def router_tile_mfma(wvs, xvs, lane, wave, red):
|
||||
"""``router_tile``'s MFMAs, K steps in order -> ``red[wave][lane]``."""
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i in range_constexpr(len(wvs)):
|
||||
acc = mfma_bf16(wvs[i].bitcast(fx.BFloat16), xvs[i].bitcast(fx.BFloat16), acc)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (slice, gate, norm) and
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_post.py (the attention's TP reduce).
|
||||
"""The mono layer's mHC seam stages, each a task table of a persistent launch
|
||||
handing off through the tagged mailbox:
|
||||
|
||||
slice (160, 32 hidden columns x 4 streams each): the owed post folded into
|
||||
the residual (R_new, bf16 out), the collapsed layer input (mailbox
|
||||
LIN) and the partial mix projection / sum of squares (mailbox PMIX)
|
||||
gate (S): the 160 partials -> post / comb (Sinkhorn) and the next pre
|
||||
norm (S): the RMSNorm -> the bf16 row, handed to the caller's store
|
||||
reduce (160): the FFN seam's attention output, the TP ranks' partials
|
||||
(pushed to every rank's ATTN region) summed in the all-reduce's
|
||||
order -> ``pend_lds``, where its slice reads it
|
||||
push (160): a rank's partial given in memory (an attention vLLM ran)
|
||||
to every rank's ATTN region, ahead of the slice's reduce
|
||||
|
||||
Every elementwise step follows aiter's ``mhc_fused_post_pre_delayed_rmsnorm``
|
||||
rounding; the mix projection and the sums of squares, split-K reductions,
|
||||
differ only in their split and order.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
batched_rounds,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
hw_exp2,
|
||||
hw_rcp,
|
||||
hw_rsq,
|
||||
lane_gather,
|
||||
rsrc,
|
||||
sum_partials,
|
||||
traced,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES, pair_layout
|
||||
from vllm.models.deepseek_v41.amd.mono.common.sync import preg
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.gemv import (
|
||||
ld_bf,
|
||||
ld_f32,
|
||||
plus,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
HC = 4
|
||||
HIDDEN = 5120
|
||||
K_MIX = HC * HIDDEN
|
||||
MIX = HC * (HC + 2) # 24 projections: pre 4 | post 4 | comb 16
|
||||
EPS = 1e-20
|
||||
HC_EPS = 1e-6
|
||||
SINKHORN = 20
|
||||
POST_MULT = 2.0
|
||||
LOG2E = 1.0 / math.log(2.0)
|
||||
|
||||
COLS = 32 # hidden columns a slice task
|
||||
SLICES = HIDDEN // COLS # 160
|
||||
KT = HC * COLS # projection columns a slice task
|
||||
PART = MIX + 1 # a slice's partials a token: the mixes, then the sum of squares
|
||||
GATE_BLOCK = 8 # slices a gate thread sums
|
||||
|
||||
|
||||
def scratch_layout(tokens: int) -> dict[str, tuple[int, int]]:
|
||||
"""Mailbox region -> (byte offset, bytes) of a seam; 8 B a pair."""
|
||||
pairs = {
|
||||
"lin": tokens * HIDDEN // 2, # bf16 pairs
|
||||
"pmix": SLICES * tokens * PART,
|
||||
}
|
||||
return pair_layout(pairs.items())
|
||||
|
||||
|
||||
def attn_peer_bytes(tokens: int, tp: int) -> int:
|
||||
"""ATTN: ``[source rank][token][hidden / 2]`` bf16 pairs."""
|
||||
return tp * tokens * HIDDEN // 2 * 8
|
||||
|
||||
|
||||
def attn_region_pair(src, t, s, col):
|
||||
return (src * s + t) * (HIDDEN // 2) + col // 2
|
||||
|
||||
|
||||
def fma(a, b, c):
|
||||
return fx.Float32(fmath.fma(a, b, c))
|
||||
|
||||
|
||||
def sigmoid(z):
|
||||
"""Triton's ``tl.sigmoid``: 1 / (1 + exp2(-z log2(e)))."""
|
||||
return fx.Float32(1.0) / (fx.Float32(1.0) + hw_exp2(z * fx.Float32(-LOG2E)))
|
||||
|
||||
|
||||
def lane_f32(v, src):
|
||||
return lane_gather(v.bitcast(fx.Int32), src).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def pend_value(c, t, col, cc):
|
||||
"""The owed post's sublayer output at (token t, column col): read from
|
||||
``pend``, or where a caller staged it (``pend_lds``, ``[token][32]`` of this
|
||||
task's columns: a reduction it did first)."""
|
||||
if const_expr(c.get("pend_lds") is None):
|
||||
return ld_bf(c["args"]["pend"], t * HIDDEN + col)
|
||||
return fx.ptr_load(c["pend_lds"] + (t * COLS + cc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- slice
|
||||
@traced
|
||||
def slice_input(c, task, idx):
|
||||
"""Thread ``idx``'s pair of the layer input (token idx / 16, columns
|
||||
32 task + 2 (idx % 16) ..)."""
|
||||
s, rl = c["S"], c["rl"]
|
||||
a = c["args"]
|
||||
c0 = task * COLS
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
cc = 2 * (idx % (COLS // 2))
|
||||
outs = []
|
||||
for d in range_constexpr(2):
|
||||
acc = fx.Float32(0.0)
|
||||
for h in range_constexpr(HC):
|
||||
rb = fx.ptr_load(rl + (t * KT + h * COLS + cc + d))
|
||||
acc = acc + ld_f32(a["pre_in"], t * HC + h) * rb
|
||||
outs.append(acc)
|
||||
c["put_bf"](c["lin"], t * HIDDEN + c0 + cc, outs)
|
||||
|
||||
|
||||
def slice_at(c0, e):
|
||||
"""Element e of a slice task at column c0: (token, stream, hidden column)."""
|
||||
return e // KT, (e % KT) // COLS, c0 + e % COLS
|
||||
|
||||
|
||||
def slice_loads(c, c0, e):
|
||||
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
if const_expr(not c.get("fold", True)):
|
||||
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
|
||||
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
|
||||
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
|
||||
return r, cm, ld_f32(a["post_in"], t * HC + o)
|
||||
|
||||
|
||||
def slice_row(c, c0, e, got):
|
||||
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
if const_expr(not c.get("fold", True)):
|
||||
# a settled residual (no owed post): stream o as it arrives
|
||||
v = got[0]
|
||||
else:
|
||||
r, cm, post = got
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
|
||||
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
|
||||
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
|
||||
for h in range_constexpr(1, HC - 1):
|
||||
v = v + cm[h] * r[h]
|
||||
# bf16 R_new is what the collapse, the sums of squares and the
|
||||
# mix projection all read
|
||||
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
|
||||
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
|
||||
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_slice(c, task):
|
||||
"""R_new, the layer input and the partial projections of hidden columns
|
||||
32 task .. +32, all four streams, every token."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
rl, fl = c["rl"], c["fl"]
|
||||
a = c["args"]
|
||||
c0 = task * COLS
|
||||
batched_rounds(
|
||||
tid,
|
||||
s * KT,
|
||||
lambda e: slice_loads(c, c0, e),
|
||||
lambda e, got: slice_row(c, c0, e, got),
|
||||
)
|
||||
for i in range_constexpr((MIX * KT + THREADS - 1) // THREADS):
|
||||
e = tid + THREADS * i
|
||||
if e < MIX * KT:
|
||||
j = e // KT
|
||||
o = (e % KT) // COLS
|
||||
# the original's two bf16 MFMAs take fn as hi = bf16(fn) and lo =
|
||||
# bf16(fn - hi): their sum, exact in fp32, is the weight it applies
|
||||
f = ld_f32(a["hc_fn"], j * K_MIX + o * HIDDEN + c0 + e % COLS)
|
||||
hi = bf16_round(f)
|
||||
fx.ptr_store(hi + bf16_round(f - hi), fl + e)
|
||||
gpu.barrier()
|
||||
# the layer input: bf16 R_new of the four streams by the incoming pre-mix,
|
||||
# products rounded, summed in order (the Triton collapse, no contraction);
|
||||
# two columns a thread, one bf16 pair a mailbox store
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
slice_input(c, task, plus(pas, tid))
|
||||
# sum of squares: token t on wave t % WAVES, two columns a lane
|
||||
for t in range_constexpr(s):
|
||||
if wave == t % WAVES:
|
||||
x0 = fx.ptr_load(rl + (t * KT + 2 * lane))
|
||||
x1 = fx.ptr_load(rl + (t * KT + 2 * lane + 1))
|
||||
sq = butterfly(x0 * x0 + x1 * x1, (32, 16, 8, 4, 2, 1))
|
||||
if lane == 0:
|
||||
c["put"](c["pmix"], (task * s + t) * PART + MIX, sq)
|
||||
# mixes: fp32 MFMA 16x16x4, A = fn rows (two 16-row tiles), B = R_new^T,
|
||||
# a token tile at a time
|
||||
for t0, n in token_tiles(s):
|
||||
slice_mix(c, task, t0, n)
|
||||
|
||||
|
||||
@traced
|
||||
def slice_mix(c, task, t0, n):
|
||||
"""The mix projection partials of tokens t0 .. t0 + n (``stage_slice``)."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
rl, fl, red = c["rl"], c["fl"], c["red"]
|
||||
tok = plus(t0, fx.min(lane % 16, n - 1))
|
||||
kw = KT // WAVES
|
||||
cs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(2)]
|
||||
for kk in range_constexpr(kw // 4):
|
||||
colk = wave * kw + lane // 16 + 4 * kk
|
||||
b = fx.ptr_load(rl + (tok * KT + colk))
|
||||
for h in range_constexpr(2):
|
||||
row = h * 16 + lane % 16
|
||||
av = fx.ptr_load(fl + (fx.min(row, MIX - 1) * KT + colk))
|
||||
av = (row < MIX).select(av, fx.Float32(0.0))
|
||||
ops = rocdl._split_mfma_operands([av, b, cs[h], 0, 0, 0])
|
||||
cs[h] = fx.Vector(rocdl.mfma_f32_16x16x4f32(T.vec(4, T.f32), *ops).result)
|
||||
for h in range_constexpr(2):
|
||||
fx.ptr_store(cs[h], red + ((wave * 2 + h) * 64 + lane) * 4)
|
||||
gpu.barrier()
|
||||
if tid < MIX * n:
|
||||
tl = tid // MIX
|
||||
t = plus(t0, tl)
|
||||
j = tid % MIX
|
||||
h = j // 16
|
||||
ln = 16 * ((j % 16) // 4) + tl
|
||||
tot = fx.Float32(0.0)
|
||||
for w in range_constexpr(WAVES):
|
||||
tot = tot + fx.ptr_load(red + (((w * 2 + h) * 64 + ln) * 4 + j % 4))
|
||||
c["put"](c["pmix"], (task * s + t) * PART + j, tot)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- gate
|
||||
def _row(v, lane, q):
|
||||
"""Comb lanes 8..23 hold ``comb[r][c]`` at lane 8 + 4 r + c: element q of
|
||||
the row."""
|
||||
return lane_f32(v, (lane - 8) // 4 * 4 + 8 + q)
|
||||
|
||||
|
||||
def _col(v, lane, q):
|
||||
return lane_f32(v, 8 + 4 * q + (lane - 8) % 4)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_gate(c, t):
|
||||
"""Token t's 160 slices, in slice order, -> post / comb / next pre."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
red = c["red"]
|
||||
a = c["args"]
|
||||
groups = SLICES // GATE_BLOCK # 20
|
||||
if tid < groups * PART:
|
||||
g = tid // PART
|
||||
j = tid % PART
|
||||
vals = c["poll"](
|
||||
[
|
||||
(c["pmix"], ((g * GATE_BLOCK + k) * s + t) * PART + j, 1)
|
||||
for k in range(GATE_BLOCK)
|
||||
]
|
||||
)
|
||||
acc = vals[0][0].bitcast(fx.Float32)
|
||||
for k in range_constexpr(1, GATE_BLOCK):
|
||||
acc = acc + vals[k][0].bitcast(fx.Float32)
|
||||
fx.ptr_store(acc, red + (g * PART + j))
|
||||
gpu.barrier()
|
||||
if wave == 0:
|
||||
j = fx.min(lane, MIX - 1)
|
||||
mix = fx.ptr_load(red + j)
|
||||
sq = fx.ptr_load(red + MIX)
|
||||
for g in range_constexpr(1, groups):
|
||||
mix = mix + fx.ptr_load(red + (g * PART + j))
|
||||
sq = sq + fx.ptr_load(red + (g * PART + MIX))
|
||||
sc = [ld_f32(a["hc_scale"], i) for i in range(3)]
|
||||
base = ld_f32(a["hc_base"], j)
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm reduce: one rstd =
|
||||
# rsq(fma(sum, 1/K, eps)), then each gate fma(mix rstd, scale, base)
|
||||
v = mix * hw_rsq(fma(sq, fx.Float32(1.0 / K_MIX), fx.Float32(EPS)))
|
||||
if lane < HC:
|
||||
pre = sigmoid(fma(v, sc[0], base))
|
||||
bo.buffer_store(pre + fx.Float32(HC_EPS), rsrc(a["pre_out"]), t * HC + lane)
|
||||
if (lane >= HC) & (lane < 2 * HC):
|
||||
post = sigmoid(fma(v, sc[1], base))
|
||||
bo.buffer_store(
|
||||
post * fx.Float32(POST_MULT), rsrc(a["post_out"]), t * HC + lane - HC
|
||||
)
|
||||
cv = fma(v, sc[2], base)
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
# Triton's exp: exp2 of (x - max) log2(e)
|
||||
m = fx.max(fx.max(row[0], row[1]), fx.max(row[2], row[3]))
|
||||
cv = hw_exp2((cv - m) * fx.Float32(LOG2E))
|
||||
# fast_dividef is x rcp(y); its first "+ eps" contracts into the fma
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
cv = fma(cv, hw_rcp((row[0] + row[1]) + (row[2] + row[3])), fx.Float32(HC_EPS))
|
||||
col = [_col(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS))
|
||||
for _ in range_constexpr(SINKHORN - 1):
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(
|
||||
((row[0] + row[1]) + (row[2] + row[3])) + fx.Float32(HC_EPS)
|
||||
)
|
||||
col = [_col(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(
|
||||
((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS)
|
||||
)
|
||||
if (lane >= 2 * HC) & (lane < MIX):
|
||||
bo.buffer_store(cv, rsrc(a["comb_out"]), t * HC * HC + lane - 2 * HC)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- norm
|
||||
@traced
|
||||
def stage_norm(c, t, store):
|
||||
"""Token t's RMSNorm (rstd = rsq(fma(sum, 1/H, eps)), bf16((x rstd) w));
|
||||
256 threads, 3 chunks of 8 at 8 t + 2048 c, each chunk's bf16 values handed
|
||||
to ``store(t, col, ys, live)`` by every thread (``live``: a thread's chunk
|
||||
in the row; threads 256.. mirror thread 255)."""
|
||||
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
|
||||
a = c["args"]
|
||||
tt = fx.min(tid, 255)
|
||||
mine = tid < 256
|
||||
cols = [tt * 8 + 2048 * ch for ch in range(3)]
|
||||
xs = []
|
||||
for ch in range_constexpr(3):
|
||||
base = t * HIDDEN + fx.min(cols[ch], HIDDEN - 8)
|
||||
words = c["poll"]([(c["lin"], base // 2 + k, 1) for k in range(4)])
|
||||
v = []
|
||||
for k in range_constexpr(4):
|
||||
v += [bf_lo(words[k][0]), bf_hi(words[k][0])]
|
||||
xs.append([(cols[ch] < HIDDEN).select(x, fx.Float32(0.0)) for x in v])
|
||||
acc = fx.Float32(0.0)
|
||||
for ch in range_constexpr(3):
|
||||
for x in xs[ch]:
|
||||
acc = acc + x * x
|
||||
acc = butterfly(acc, (1, 2, 4, 8, 16, 32))
|
||||
if (lane == 0) & mine:
|
||||
fx.ptr_store(acc, red + wave)
|
||||
gpu.barrier()
|
||||
tot = butterfly(fx.ptr_load(red + lane % 4), (1, 2))
|
||||
r = hw_rsq(fma(tot, fx.Float32(1.0 / HIDDEN), fx.Float32(EPS)))
|
||||
for ch in range_constexpr(3):
|
||||
col = fx.min(cols[ch], HIDDEN - 8)
|
||||
ys = [
|
||||
bf16_round((xs[ch][j] * r) * ld_bf(a["norm_w"], col + j)) for j in range(8)
|
||||
]
|
||||
store(t, col, ys, mine & (cols[ch] < HIDDEN))
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reduce
|
||||
@traced
|
||||
def stage_reduce(c, task):
|
||||
"""The attention output at this slice's 32 columns: the TP ranks' partials
|
||||
in the all-reduce's order (``sum_partials``), in fp32 -> bf16 ->
|
||||
``pend_lds``."""
|
||||
s, tid = c["S"], c["tid"]
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
reduce_attn_pair(c, task, plus(pas, tid))
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
@traced
|
||||
def reduce_attn_pair(c, task, idx):
|
||||
"""Thread ``idx``'s column pair of token idx / 16 (``stage_reduce``)."""
|
||||
s = c["S"]
|
||||
c0 = task * COLS
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
col = c0 + 2 * (idx % (COLS // 2))
|
||||
own = preg(c["sym"], 0, "attn")
|
||||
acc0, acc1 = sum_partials(
|
||||
c["poll"],
|
||||
own,
|
||||
lambda src: attn_region_pair(src, t, s, col),
|
||||
c["d"].tp,
|
||||
)
|
||||
lds = c["pend_lds"] + (t * COLS + col - c0)
|
||||
fx.ptr_store(bf16_round(acc0), lds)
|
||||
fx.ptr_store(bf16_round(acc1), lds + 1)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_push(c, task, part):
|
||||
"""This rank's attention partial (``part``: bf16 [S, HIDDEN], wo_b's
|
||||
unreduced output) at this slice's 32 columns, every token, to every rank's
|
||||
ATTN region: the pairs ``stage_reduce`` polls, as the bits vLLM wrote."""
|
||||
s, tid, tp = c["S"], c["tid"], c["d"].tp
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
idx = plus(pas, tid)
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
col = task * COLS + 2 * (idx % (COLS // 2))
|
||||
w = bo.buffer_load(
|
||||
rsrc(part), (t * HIDDEN + col) // 2, vec_width=1, dtype=T.i32
|
||||
)
|
||||
pair = attn_region_pair(c["rank"], t, s, col)
|
||||
for p in range_constexpr(tp):
|
||||
c["put_words"](preg(c["peer_addr"](p), 0, "attn"), pair, [w])
|
||||
@@ -0,0 +1,435 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
|
||||
|
||||
A decode step's backbone layer -- the attention seam, the attention, its TP
|
||||
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
|
||||
FlyDSL launches (``mono``, this directory) on the layer's loaded
|
||||
weights, the TP reductions inside the kernels. Every backbone layer of the
|
||||
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
|
||||
the first layer and the Engram layers hand K1 a settled residual, and an index
|
||||
source runs vLLM's compressor and indexer between the two launches. A layer
|
||||
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
|
||||
and attention and runs the rest as one launch, the FFN launch, from wo_b's
|
||||
unreduced output. A step takes the kernels when it is decode only, at most
|
||||
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
|
||||
a whole layer also needs causal SWA windows. Anything else runs the layer as
|
||||
before; the paths share every tensor at the layer boundary, so they interleave
|
||||
freely.
|
||||
|
||||
Every TP rank must take the same path at every layer (the kernels wait on each
|
||||
other's pushes): the conditions read only metadata every rank holds alike.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV4ROCMAiterSparseSWAMetadata
|
||||
from vllm.models.deepseek_v41.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RECORD = 584 # an fp8_ds_mla KV record: 576 data bytes, 8 scale bytes
|
||||
SWA_WIDTH = 128 # a causal decode token's window slots
|
||||
MAX_ROWS = 48 # the kernels' step rows at most: 8 requests x (1 + 5 DSpark drafts)
|
||||
# the kernels' ISA: CDNA4 (gfx950) scaled MFMA and MX formats
|
||||
CDNA_VERSIONS = (4,)
|
||||
|
||||
_runner = None
|
||||
|
||||
|
||||
def _mono_runner(device: torch.device):
|
||||
"""The process's mono runner (one a TP rank, shared by every layer). Its
|
||||
peer-memory handshake is collective over the TP group: every rank builds it
|
||||
at the same layer of the same step."""
|
||||
global _runner
|
||||
if _runner is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
# its peer memory is allocated and exchanged eagerly; vLLM warms
|
||||
# every graph up eagerly before capturing it
|
||||
raise RuntimeError(
|
||||
"--monokernel was first reached inside a CUDA graph "
|
||||
"capture, before an eager decode step."
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
|
||||
|
||||
assert MAX_TOKENS >= MAX_ROWS, MAX_TOKENS
|
||||
_runner = DSV41MonoLayer(
|
||||
get_tensor_model_parallel_world_size(),
|
||||
get_tensor_model_parallel_rank(),
|
||||
get_tp_group().cpu_group,
|
||||
device,
|
||||
)
|
||||
return _runner
|
||||
|
||||
|
||||
def _record_view(kv: torch.Tensor, block: int) -> torch.Tensor:
|
||||
"""A layer's fp8_ds_mla cache as [blocks, block, RECORD] bytes (a block's
|
||||
data rows, then its scale words: the kernels index both from it)."""
|
||||
kv = kv if kv.dtype == torch.uint8 else kv.view(torch.uint8)
|
||||
assert kv.stride(0) >= block * RECORD, (tuple(kv.shape), kv.stride(), block)
|
||||
return torch.as_strided(kv, (kv.shape[0], block, RECORD), (kv.stride(0), RECORD, 1))
|
||||
|
||||
|
||||
def _scale_bytes(scale: torch.Tensor) -> torch.Tensor:
|
||||
return scale if scale.dtype == torch.uint8 else scale.view(torch.uint8)
|
||||
|
||||
|
||||
def _compute_units() -> int:
|
||||
return current_platform.num_compute_units(torch.accelerator.current_device_index())
|
||||
|
||||
|
||||
def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None:
|
||||
"""Why the kernels cannot run this deployment's MoE, or None. They read every
|
||||
one of the 384 experts on each rank, TP-sharded, in AITER's A8W4 MXFP4
|
||||
layout."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
|
||||
pc = vllm_config.parallel_config
|
||||
if pc.enable_expert_parallel or pc.enable_eplb or pc.data_parallel_size > 1:
|
||||
return "needs tensor parallelism only: no expert or data parallelism"
|
||||
if envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
|
||||
return "needs AITER's A8W4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 unset)"
|
||||
experts = getattr(getattr(layer.ffn, "experts", None), "routed_experts", None)
|
||||
if experts is None:
|
||||
return None # not a routed MoE: the layer checks decline it
|
||||
backend = getattr(experts.quant_method, "mxfp4_backend", None)
|
||||
if backend != Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
return f"needs the AITER_MXFP4_BF16 MoE backend, not {backend}"
|
||||
return None
|
||||
|
||||
|
||||
def _select(
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
positions: torch.Tensor,
|
||||
normed: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
) -> None:
|
||||
"""An index source's compressor and top-k selection, from K1's normed input
|
||||
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
|
||||
indexer op, serial."""
|
||||
attn = layer.attn
|
||||
comp, indexer = attn.compressor, attn.indexer
|
||||
assert indexer is not None
|
||||
if comp is not None:
|
||||
kv_score = torch.mm(
|
||||
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
|
||||
)
|
||||
latent = comp(kv_score, positions)
|
||||
comp.insert_cache(latent, positions, attn.rotary_emb)
|
||||
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
|
||||
weights, _ = indexer.weights_proj(normed)
|
||||
q, q_scale, weights = indexer.forward_q(
|
||||
qr, None, weights, positions, attn.indexer_rotary_emb
|
||||
)
|
||||
assert weights is not None
|
||||
indexer.indexer_op(
|
||||
normed, (q, q_scale) if q_scale is not None else q, None, weights
|
||||
)
|
||||
|
||||
|
||||
class MonoDecodeLayer:
|
||||
"""One decoder layer's mono path (``create``: None when the layer is not
|
||||
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
|
||||
launch after vLLM's attention (``ffn``). Its weights are taken from the
|
||||
layer at the first call, after loading."""
|
||||
|
||||
def __init__(self, ffn_only: bool = False) -> None:
|
||||
self.ffn_only = ffn_only
|
||||
self._weights: MonoLayerWeights | None = None
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
layer: "DeepseekV4DecoderLayer", vllm_config
|
||||
) -> "MonoDecodeLayer | None":
|
||||
if not vllm_config.kernel_config.monokernel:
|
||||
return None
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
# the deployment: an explicit opt-in that cannot run is an error
|
||||
why = None
|
||||
if get_cdna_version() not in CDNA_VERSIONS:
|
||||
why = f"needs CDNA{'/'.join(map(str, CDNA_VERSIONS))}"
|
||||
elif get_tensor_model_parallel_world_size() not in (2, 4):
|
||||
why = "needs tensor parallel size 2 or 4"
|
||||
else:
|
||||
why = _unsupported_moe(layer, vllm_config)
|
||||
if why is None:
|
||||
try:
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import BLOCKS
|
||||
except ImportError as err:
|
||||
why = f"needs FlyDSL and AITER's FlyDSL helpers ({err})"
|
||||
else:
|
||||
# every CTA of a launch stays resident: a partitioned GPU deadlocks
|
||||
cus = _compute_units()
|
||||
if cus < BLOCKS:
|
||||
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
|
||||
if why is not None:
|
||||
raise ValueError(f"--monokernel {why}.")
|
||||
# the layer: the kernels serve the backbone's seams and MoE
|
||||
config = vllm_config.model_config.hf_config
|
||||
attn, ffn = layer.attn, layer.ffn
|
||||
if layer.use_sequence_parallel:
|
||||
why = "sequence-parallel layer"
|
||||
elif not layer.fuse_seam_norm:
|
||||
why = "needs aiter's fused mHC seam"
|
||||
elif attn.layer_id >= config.num_hidden_layers:
|
||||
why = "draft layer"
|
||||
elif (
|
||||
ffn.shared_experts is None
|
||||
or ffn.gate.tid2eid is not None
|
||||
or ffn.n_routed_experts != 384
|
||||
or ffn.n_activated_experts != 6
|
||||
or ffn.routed_scaling_factor != 1.5
|
||||
or ffn.swiglu_limit != 10.0
|
||||
or ffn.scoring_func != "sqrtsoftplus"
|
||||
or not ffn.renormalize
|
||||
):
|
||||
why = "MoE routing outside the kernels' (384 / 6, sqrtsoftplus, x1.5)"
|
||||
if why is not None:
|
||||
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
|
||||
return None
|
||||
# the whole layer: the kernels' attention takes the standard layers; an
|
||||
# index source's compressor and indexer run between its two launches
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
if attn.compressor is not None and attn.indexer is None:
|
||||
why = "a compressor without an indexer"
|
||||
elif attn.indexer is not None and not isinstance(
|
||||
attn.indexer, DeepseekV41RocmMxfp4Indexer
|
||||
):
|
||||
why = f"a {type(attn.indexer).__name__}"
|
||||
elif attn.compress_ratio not in (0, 1, 2):
|
||||
why = f"compress ratio {attn.compress_ratio}"
|
||||
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
|
||||
why = f"a {attn.kv_cache_dtype} KV cache"
|
||||
if why is not None:
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode: the FFN launch for layers with %s", why
|
||||
)
|
||||
return MonoDecodeLayer(ffn_only=True)
|
||||
logger.info_once("DSv4.1 mono decode: the whole standard backbone layers")
|
||||
return MonoDecodeLayer()
|
||||
|
||||
def weights(self, layer: "DeepseekV4DecoderLayer"):
|
||||
if self._weights is not None:
|
||||
return self._weights
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
|
||||
|
||||
a, f = layer.attn, layer.ffn
|
||||
e, sh = f.experts.routed_experts, f.shared_experts
|
||||
assert sh is not None # ``create`` took only layers with one
|
||||
self._weights = MonoLayerWeights(
|
||||
attn=None if self.ffn_only else self._attn_weights(a),
|
||||
hc_attn_fn=layer.hc_attn_fn,
|
||||
hc_attn_scale=layer.hc_attn_scale,
|
||||
hc_attn_base=layer.hc_attn_base,
|
||||
attn_norm=layer.attn_norm.weight,
|
||||
hc_ffn_fn=layer.hc_ffn_fn,
|
||||
hc_ffn_scale=layer.hc_ffn_scale,
|
||||
hc_ffn_base=layer.hc_ffn_base,
|
||||
ffn_norm=layer.ffn_norm.weight,
|
||||
gate_w=f.gate.weight,
|
||||
bias=f.gate.e_score_correction_bias,
|
||||
w13=e.w13_weight,
|
||||
w13_s=e.w13_weight_scale,
|
||||
w2=e.w2_weight,
|
||||
w2_s=e.w2_weight_scale,
|
||||
sgu=sh.gate_up_proj.weight,
|
||||
sgu_s=_scale_bytes(sh.gate_up_proj.weight_scale),
|
||||
sw2=sh.down_proj.weight,
|
||||
sw2_s=_scale_bytes(sh.down_proj.weight_scale),
|
||||
)
|
||||
self._weights.check(get_tensor_model_parallel_world_size())
|
||||
return self._weights
|
||||
|
||||
@staticmethod
|
||||
def _attn_weights(a):
|
||||
from vllm.models.deepseek_v41.amd.mono.attention.plan import Dims
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import AttnWeights
|
||||
|
||||
attn = AttnWeights(
|
||||
layer_id=a.layer_id,
|
||||
wqkv=a.fused_wqa_wkv.weight,
|
||||
wqkv_scale=_scale_bytes(a.fused_wqa_wkv.weight_scale),
|
||||
q_norm=a.q_norm.weight,
|
||||
kv_norm=a.kv_norm.weight,
|
||||
wq_b=a.wq_b.weight,
|
||||
wq_b_scale=_scale_bytes(a.wq_b.weight_scale),
|
||||
wo_a=a.wo_a.weight,
|
||||
wo_a_scale=_scale_bytes(a.wo_a.weight_scale),
|
||||
wo_b=a.wo_b.weight,
|
||||
wo_b_scale=_scale_bytes(a.wo_b.weight_scale),
|
||||
attn_sink=a.attn_sink,
|
||||
cos_sin=a.rotary_emb.cos_sin_cache,
|
||||
ratio=a.compress_ratio,
|
||||
)
|
||||
attn.check(Dims(get_tensor_model_parallel_world_size()))
|
||||
return attn
|
||||
|
||||
@staticmethod
|
||||
def _skip(why: str) -> None:
|
||||
logger.debug_once("DSv4.1 mono decode skipped a step: %s", why)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _decode_step(layer: "DeepseekV4DecoderLayer", rows: int):
|
||||
"""(why this step takes the original path -- None for a decode-only step
|
||||
the kernels take -- the layer's SWA metadata, its compressed cache's)."""
|
||||
if not 1 <= rows <= MAX_ROWS:
|
||||
return f"{rows} rows", None, None
|
||||
if not is_forward_context_available():
|
||||
return "no forward context", None, None
|
||||
fc = get_forward_context()
|
||||
md = fc.attn_metadata
|
||||
attn = layer.attn
|
||||
swa, comp = None, None
|
||||
if isinstance(md, dict):
|
||||
swa = cast(
|
||||
"DeepseekV4ROCMAiterSparseSWAMetadata | None",
|
||||
md.get(attn.swa_cache_layer.prefix),
|
||||
)
|
||||
if attn.compressed_cache_prefix is not None:
|
||||
comp = cast(
|
||||
"DeepseekV4FlashMLAMetadata | None",
|
||||
md.get(attn.compressed_cache_prefix),
|
||||
)
|
||||
why = None
|
||||
if fc.cudagraph_runtime_mode not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL):
|
||||
# a PIECEWISE (breakable) capture is replayed for mixed batches too
|
||||
why = f"{fc.cudagraph_runtime_mode} step"
|
||||
elif not isinstance(md, dict): # a profile / dummy run, or DBO ubatches
|
||||
why = "no attention metadata"
|
||||
elif swa is None:
|
||||
why = "layer metadata missing"
|
||||
elif swa.num_prefills != 0 or swa.num_decodes == 0:
|
||||
why = "not a decode-only step"
|
||||
elif swa.num_decode_tokens > rows:
|
||||
why = "more decode tokens than rows"
|
||||
return why, swa, comp
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor | None,
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, ...] | None:
|
||||
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
|
||||
mono kernels, or None when this step takes the original path (always for
|
||||
an ``ffn_only`` layer)."""
|
||||
if self.ffn_only:
|
||||
return None
|
||||
first = residual is None
|
||||
if first and x.dim() != 2:
|
||||
return self._skip("first layer of a pipeline stage")
|
||||
why, swa, comp = self._decode_step(layer, x.shape[0])
|
||||
attn = layer.attn
|
||||
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
|
||||
if why is None:
|
||||
# the kernels' attention: causal SWA windows, the compressed cache
|
||||
if comp is None and attn.compress_ratio:
|
||||
why = "layer metadata missing"
|
||||
elif swa.decode_swa_width != SWA_WIDTH:
|
||||
why = f"SWA width {swa.decode_swa_width}"
|
||||
elif (
|
||||
swa.decode_swa_indices is None
|
||||
or swa.decode_swa_lens is None
|
||||
or swa.token_to_req_indices is None
|
||||
):
|
||||
why = "SWA decode metadata missing"
|
||||
elif not all(t is not None and t.is_contiguous() for t in tensors):
|
||||
why = "non-contiguous layer inputs"
|
||||
if why is not None:
|
||||
return self._skip(why)
|
||||
assert swa is not None
|
||||
runner = _mono_runner(x.device)
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
|
||||
)
|
||||
# K1's seam folds the owed post into the residual, except where the
|
||||
# residual arrives settled: the embedding, or after the Engram write
|
||||
fold = True
|
||||
if first:
|
||||
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
|
||||
pre_mix = runner.first_pre_mix(x.shape[0])
|
||||
post_mix = res_mix = pre_mix # unread without the fold
|
||||
fold = False
|
||||
elif layer.engram is not None and engram_hashes is not None:
|
||||
residual = layer.mhc_post(x, residual, post_mix, res_mix)
|
||||
hashes = engram_hashes[:, layer.engram.layer_hash_index]
|
||||
residual = layer.engram(residual, hashes, engram_mask)
|
||||
fold = False
|
||||
kw: dict[str, Any] = {}
|
||||
if attn.indexer is not None:
|
||||
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
|
||||
if attn.compress_ratio:
|
||||
assert comp is not None
|
||||
kw.update(
|
||||
topk_indices=attn.topk_indices_buffer,
|
||||
comp_cache=_record_view(
|
||||
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
|
||||
),
|
||||
comp_block_table=comp.block_table,
|
||||
)
|
||||
return runner.forward(
|
||||
self.weights(layer),
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
positions,
|
||||
swa.slot_mapping,
|
||||
_record_view(attn.swa_cache_layer.kv_cache, swa.block_size),
|
||||
swa.decode_swa_indices,
|
||||
swa.decode_swa_lens,
|
||||
swa.token_to_req_indices,
|
||||
fold=fold,
|
||||
**kw,
|
||||
)
|
||||
|
||||
def ffn(
|
||||
self,
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
part: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_mix: torch.Tensor,
|
||||
res_mix: torch.Tensor,
|
||||
pre_mix: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...] | None:
|
||||
"""The layer's outputs by the FFN launch from ``part``, wo_b's unreduced
|
||||
output, and the attention seam's outputs; None when this step takes the
|
||||
original path (the caller then reduces ``part`` itself)."""
|
||||
why, _, _ = self._decode_step(layer, part.shape[0])
|
||||
tensors = (part, residual, post_mix, res_mix, pre_mix)
|
||||
if why is None and not all(t.is_contiguous() for t in tensors):
|
||||
why = "non-contiguous layer inputs"
|
||||
if why is not None:
|
||||
return self._skip(why)
|
||||
runner = _mono_runner(part.device)
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode FFN launch active (decode steps of <= %d rows)",
|
||||
MAX_ROWS,
|
||||
)
|
||||
return runner.ffn(self.weights(layer), *tensors)
|
||||
@@ -8,6 +8,7 @@ from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
@@ -41,6 +42,7 @@ from vllm.platforms.rocm import _ON_GFX950
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.multi_stream_utils import execute_in_parallel
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
@@ -61,6 +63,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
|
||||
rocm_paged_mxfp4_cache_layout,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -285,7 +288,35 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
||||
prefill_swa_ragged_indptr: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV41ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.enable_adaptive_verification:
|
||||
# Per-token request ownership and compressed slot mappings are built
|
||||
# from device query boundaries into persistent buffers.
|
||||
return AttentionCGSupport.ALWAYS
|
||||
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBuilder):
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.enable_adaptive_verification:
|
||||
# SWA indices, lengths, and request ownership are built from device
|
||||
# query boundaries and copied into persistent ragged buffers.
|
||||
return AttentionCGSupport.ALWAYS
|
||||
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
|
||||
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
|
||||
# also refreshes the ROCm-specific ragged SWA indices and indptrs.
|
||||
supports_draft_decode_metadata_update = False
|
||||
@@ -508,8 +539,8 @@ class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
return [64, 128]
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4SparseMLAMetadataBuilder]:
|
||||
return DeepseekV4SparseMLAMetadataBuilder
|
||||
def get_builder_cls() -> type[DeepseekV41ROCMAiterMLASparseMetadataBuilder]:
|
||||
return DeepseekV41ROCMAiterMLASparseMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV41ROCMAiterSparseSWABackend(DeepseekSparseSWABackend):
|
||||
@@ -929,7 +960,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
|
||||
def _wo_b_after_wo_a(self, zf: torch.Tensor) -> torch.Tensor:
|
||||
if self._wo_b_scale is not None and zf.dim() == 2:
|
||||
return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True)
|
||||
return self._bpre_attn_gemm(
|
||||
self.wo_b.weight, self._wo_b_scale, zf, self.wo_b.reduce_results
|
||||
)
|
||||
return self.wo_b(zf)
|
||||
|
||||
def forward_mqa(
|
||||
|
||||
@@ -51,6 +51,7 @@ from ..common.mm_preprocess import (
|
||||
DeepseekV4VLProcessingInfo,
|
||||
)
|
||||
from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin
|
||||
from ..decoder_replay_layers import DecoderReplayLayers
|
||||
from .model import (
|
||||
DeepseekV41LLMForCausalLM,
|
||||
_linear_scale_param_name,
|
||||
@@ -299,6 +300,10 @@ class DeepseekV41ForCausalLM(
|
||||
def token_lookback_depth(self) -> int:
|
||||
return self.language_model.token_lookback_depth
|
||||
|
||||
@property
|
||||
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
|
||||
return self.language_model.decoder_replay_layers
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
@@ -356,10 +357,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
|
||||
# Sink padded to the same head count, initialized to -inf (no sink
|
||||
# effect). Weight loading fills the first n_local_heads slots.
|
||||
self.attn_sink = nn.Parameter(
|
||||
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
|
||||
|
||||
self.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
@@ -547,11 +545,11 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
swa_bounded_replay = False
|
||||
if swa_bounded_replay and current_platform.is_rocm():
|
||||
logger.warning_once(
|
||||
"SWA bounded replay is off on ROCm (the sparse SWA metadata "
|
||||
"builders forward replay_start, but the window clamp it relies "
|
||||
"on lives in the FlashInfer and FlashMLA prefill kernels, so "
|
||||
"the padded slots fault); the sliding-window cache takes part "
|
||||
"in prefix caching instead."
|
||||
"Prefix-cache SWA replay is off on ROCm (the sparse SWA "
|
||||
"metadata builders forward replay_start, but the window clamp "
|
||||
"it relies on lives in the FlashInfer and FlashMLA prefill "
|
||||
"kernels, so the padded slots fault); the sliding-window cache "
|
||||
"takes part in prefix caching instead."
|
||||
)
|
||||
swa_bounded_replay = False
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonWarmupTensor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
@@ -1012,7 +1013,7 @@ def _combine_topk_swa_warmup_inputs(vllm_config: Any) -> _WarmupInputRows:
|
||||
"""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
compress_ratios = getattr(hf_config, "compress_ratios", None) or [0]
|
||||
compress_ratios = get_compress_ratios(hf_config) or [0]
|
||||
topk = _hf_config_int(vllm_config, "index_topk", 0)
|
||||
return zip_inputs(
|
||||
*(
|
||||
|
||||
@@ -17,6 +17,8 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
from . import MXFP4_BLOCK_SIZE, _fp32x2_to_fp4x2
|
||||
|
||||
@@ -51,7 +53,8 @@ def indexer_k_norm_rope_store(
|
||||
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
|
||||
compress_ratio: group size; keys are emitted at group boundaries.
|
||||
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
|
||||
per-token FP8 with a single fp32 scale.
|
||||
per-token FP8 with a single fp32 scale (four packed ue8m0 bytes
|
||||
under `mqa_logits_use_mx_scales`).
|
||||
|
||||
"""
|
||||
num_tokens = kv_slot_mapping.numel()
|
||||
@@ -111,6 +114,7 @@ def indexer_k_norm_rope_store(
|
||||
KV_BLOCK_STRIDE=k_cache.stride(0),
|
||||
FP8_MAX=448.0,
|
||||
USE_FP4=use_fp4_cache,
|
||||
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
SHUFFLE=shuffle,
|
||||
BLOCK_TILE_SIZE=_BLOCK_TILE_SIZE,
|
||||
HEAD_TILE_SIZE=_HEAD_TILE_SIZE,
|
||||
@@ -139,6 +143,7 @@ def _indexer_k_norm_rope_quant_store_kernel(
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FP4: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
SHUFFLE: tl.constexpr,
|
||||
BLOCK_TILE_SIZE: tl.constexpr,
|
||||
HEAD_TILE_SIZE: tl.constexpr,
|
||||
@@ -252,4 +257,10 @@ def _indexer_k_norm_rope_quant_store_kernel(
|
||||
tl.store(val_ptr + tiled, x_uint8)
|
||||
else:
|
||||
tl.store(val_ptr + block, x_uint8)
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
scale_ptr.to(tl.pointer_type(tl.int32)),
|
||||
pack_ue8m0_x4(tl.exp2(exponent)),
|
||||
)
|
||||
else:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.forward_context import get_forward_context, override_forward_context
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -56,8 +56,9 @@ class DecoderReplayCudaGraphManager(CudaGraphManager):
|
||||
config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype
|
||||
assert isinstance(act, torch.dtype)
|
||||
hc, h, f32 = config.hc_mult, config.hidden_size, torch.float32
|
||||
mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
ids = torch.int64 if mega_moe else torch.int32
|
||||
# DeepseekV4Model.forward casts input_ids to int64 for native MegaMoE only.
|
||||
native = vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
ids = torch.int64 if native else torch.int32
|
||||
# hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual
|
||||
shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)]
|
||||
dtypes = [act, torch.int64, ids, f32, f32, f32, act]
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -44,7 +40,7 @@ from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
@@ -409,12 +405,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -431,6 +421,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
else f".{self.linear_scale_name}"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts." in name:
|
||||
experts = self.model.layers[extract_layer_index(name)].ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
@@ -476,11 +473,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
@@ -492,17 +484,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
self.process_weights_after_loading()
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def _finalize_moe(self) -> None:
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self._finalize_moe()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
|
||||
@@ -206,24 +206,36 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
|
||||
metadata = super().build(
|
||||
common_prefix_len, common_attn_metadata, fast_build, replay_start
|
||||
)
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
if not common_attn_metadata.causal and num_tokens > 0:
|
||||
assert metadata.decode_swa_lens is not None
|
||||
assert metadata.seq_lens is not None
|
||||
assert metadata.token_to_req_indices is not None
|
||||
topk_lens = self._decode_topk_lens[:num_tokens]
|
||||
seq_lens = self._decode_seq_lens[:num_tokens]
|
||||
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
|
||||
torch.index_select(
|
||||
metadata.seq_lens,
|
||||
0,
|
||||
metadata.token_to_req_indices[:num_tokens],
|
||||
out=seq_lens,
|
||||
)
|
||||
metadata.flashinfer_decode_topk_lens = topk_lens
|
||||
metadata.flashinfer_decode_seq_lens = seq_lens
|
||||
if not common_attn_metadata.causal and metadata.num_decode_tokens > 0:
|
||||
self._fill_noncausal_decode_lens(metadata)
|
||||
return metadata
|
||||
|
||||
def update_draft_decode_metadata(
|
||||
self, metadata: "DeepseekSparseSWAMetadata"
|
||||
) -> None:
|
||||
super().update_draft_decode_metadata(metadata)
|
||||
if metadata.flashinfer_decode_topk_lens is not None:
|
||||
self._fill_noncausal_decode_lens(metadata)
|
||||
|
||||
def _fill_noncausal_decode_lens(
|
||||
self, metadata: "DeepseekSparseSWAMetadata"
|
||||
) -> None:
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
assert metadata.decode_swa_lens is not None
|
||||
assert metadata.seq_lens is not None
|
||||
assert metadata.token_to_req_indices is not None
|
||||
topk_lens = self._decode_topk_lens[:num_tokens]
|
||||
seq_lens = self._decode_seq_lens[:num_tokens]
|
||||
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
|
||||
torch.index_select(
|
||||
metadata.seq_lens,
|
||||
0,
|
||||
metadata.token_to_req_indices[:num_tokens],
|
||||
out=seq_lens,
|
||||
)
|
||||
metadata.flashinfer_decode_topk_lens = topk_lens
|
||||
metadata.flashinfer_decode_seq_lens = seq_lens
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
|
||||
@staticmethod
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_engram_dp_size,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
@@ -1251,14 +1250,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1275,6 +1266,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# Vision weights are loaded by the outer multimodal wrapper.
|
||||
logger.warning_once("Skipping non-text weight: %s", name)
|
||||
continue
|
||||
if ".shared_experts." in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
experts = self.layers[extract_layer_index(name)].ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = self._pad_shared_expert_weight(
|
||||
self.quant_config, name, loaded_weight
|
||||
@@ -1337,14 +1337,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
@@ -1406,10 +1398,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def finalize_mhc_broadcast_weights(self) -> None:
|
||||
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
|
||||
return
|
||||
@@ -1680,7 +1668,6 @@ class DeepseekV41LLMForCausalLM(
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model.finalize_mega_moe_weights()
|
||||
self.model.finalize_mhc_broadcast_weights()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
|
||||
@@ -337,7 +337,7 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
self._warm_up_replay_kernels(input_batch, slot_mappings)
|
||||
window, cacheable_groups = self._replay
|
||||
replay_start: torch.Tensor | None = None
|
||||
if window:
|
||||
if window or self.decoder_replay_layers is not None:
|
||||
num_reqs = input_batch.num_reqs
|
||||
# Decode rows sit above the hit, so only prefills carry a replay
|
||||
# start; dummy batches (captures, profiling) carry none.
|
||||
@@ -349,7 +349,7 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
replay_start = self._replay_start_staging.copy_to_gpu(
|
||||
replay_start_np, out=self._replay_start[:num_reqs]
|
||||
)
|
||||
if replay_start_np.any():
|
||||
if window and replay_start_np.any():
|
||||
# The replayed tokens rebuild window KV only: their slots in the
|
||||
# prefix-cacheable groups are padded so the cached KV stays as is.
|
||||
_pad_replayed_slots_kernel[(num_reqs,)](
|
||||
@@ -397,21 +397,21 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
dummy batches never replay or trim, so they would not launch them."""
|
||||
assert self._replay is not None
|
||||
window, cacheable_groups = self._replay
|
||||
if not window:
|
||||
return
|
||||
_pad_replayed_slots_kernel.warmup(
|
||||
slot_mappings,
|
||||
slot_mappings.stride(0),
|
||||
cacheable_groups,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.positions,
|
||||
self._replay_start,
|
||||
window,
|
||||
PAD_SLOT_ID,
|
||||
NUM_GROUPS=cacheable_groups.numel(),
|
||||
BLOCK=1024,
|
||||
grid=(1,),
|
||||
)
|
||||
# Pad warmup is prefix-cache SWA. A zero window must not skip the gather warmup.
|
||||
if window:
|
||||
_pad_replayed_slots_kernel.warmup(
|
||||
slot_mappings,
|
||||
slot_mappings.stride(0),
|
||||
cacheable_groups,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.positions,
|
||||
self._replay_start,
|
||||
window,
|
||||
PAD_SLOT_ID,
|
||||
NUM_GROUPS=cacheable_groups.numel(),
|
||||
BLOCK=1024,
|
||||
grid=(1,),
|
||||
)
|
||||
if self.decoder_replay_layers is None:
|
||||
return
|
||||
assert self._kept_slot_mappings is not None
|
||||
@@ -669,7 +669,7 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
copy.create_metadata_builders(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
group.metadata_builders[0].kernel_block_size,
|
||||
group.kernel_block_size,
|
||||
)
|
||||
copies.append(copy)
|
||||
own_groups.append(copies)
|
||||
|
||||
+116
-241
@@ -228,13 +228,13 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
|
||||
NC: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_t, i_c, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
i_i, i_j = i_c // NC, i_c % NC
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -255,58 +255,51 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
|
||||
A += (bos * H + i_h) * BT
|
||||
Aqk += (bos * H + i_h) * BT
|
||||
|
||||
p_b = tl.make_block_ptr(
|
||||
beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)
|
||||
)
|
||||
b_b = tl.load(p_b, boundary_check=(0,))
|
||||
o_i = i_t * BT + i_i * BC + tl.arange(0, BC)
|
||||
o_j = i_t * BT + i_j * BC + tl.arange(0, BC)
|
||||
m_i = o_i < T
|
||||
m_j = o_j < T
|
||||
p_b = beta + bos * H + i_h + o_i * H
|
||||
b_b = tl.load(p_b, mask=m_i, other=0.0)
|
||||
|
||||
b_A = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
b_kt = tl.make_block_ptr(
|
||||
k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
|
||||
)
|
||||
p_gk = tl.make_block_ptr(
|
||||
g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
|
||||
)
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_qk = m_i[:, None] & m_k[None, :]
|
||||
m_kj = m_k[:, None] & m_j[None, :]
|
||||
|
||||
p_q = q + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
b_kt = k + o_k[:, None] + o_j[None, :] * (H * K)
|
||||
p_gk = g + o_k[:, None] + o_j[None, :] * (H * K)
|
||||
|
||||
# [BK,]
|
||||
b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
|
||||
# [BC, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
b_k = tl.load(p_k, mask=m_qk, other=0.0) * exp2(b_g - b_gn[None, :])
|
||||
# [BK, BC]
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kt = tl.load(b_kt, boundary_check=(0, 1))
|
||||
b_gk = tl.load(p_gk, mask=m_kj, other=0.0)
|
||||
b_kt = tl.load(b_kt, mask=m_kj, other=0.0)
|
||||
# [BC, BC]
|
||||
b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
|
||||
b_A += tl.dot(b_k, b_ktg)
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
|
||||
b_Aqk += tl.dot(b_qg, b_ktg)
|
||||
|
||||
b_A *= b_b[:, None]
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_Aqk = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
o_jA = i_j * BC + tl.arange(0, BC)
|
||||
m_A = m_i[:, None] & (o_jA[None, :] < BT)
|
||||
p_A = A + o_i[:, None] * (H * BT) + o_jA[None, :]
|
||||
tl.store(p_A, b_A.to(A.dtype.element_ty), mask=m_A)
|
||||
p_Aqk = Aqk + o_i[:, None] * (H * BT) + o_jA[None, :]
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_A)
|
||||
|
||||
|
||||
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
|
||||
@@ -333,12 +326,12 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -357,33 +350,14 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
|
||||
m_A = (i_t * BT + i_i * BC + o_i) < T
|
||||
o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
|
||||
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
o_c = i_t * BT + i_i * BC + tl.arange(0, BC)
|
||||
m_qk = m_A[:, None] & m_k[None, :]
|
||||
p_q = q + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_k = tl.load(p_k, mask=m_qk, other=0.0)
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
|
||||
p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
|
||||
b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]
|
||||
@@ -536,12 +510,12 @@ def recompute_w_u_fwd_kernel(
|
||||
IS_VARLEN: tl.constexpr,
|
||||
DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -550,108 +524,57 @@ def recompute_w_u_fwd_kernel(
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,))
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
p_b = beta + bos * H + i_h + o_t * H
|
||||
b_b = tl.load(p_b, mask=m_t, other=0.0)
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
o_A = tl.arange(0, BT)
|
||||
m_A = m_t[:, None] & (o_A[None, :] < BT)
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
|
||||
for i_v in range(tl.cdiv(V, BV)):
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_u = tl.make_block_ptr(
|
||||
u + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
m_v = m_t[:, None] & (o_v[None, :] < V)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
b_v = tl.load(p_v, mask=m_v, other=0.0)
|
||||
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
|
||||
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
|
||||
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_w = tl.make_block_ptr(
|
||||
w + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_tk = m_t[:, None] & m_k[None, :]
|
||||
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_k = tl.load(p_k, mask=m_tk, other=0.0)
|
||||
b_kb = b_k * b_b[:, None]
|
||||
|
||||
p_gk = tl.make_block_ptr(
|
||||
gk + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_qg = tl.make_block_ptr(
|
||||
qg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_q = tl.load(p_q, mask=m_tk, other=0.0)
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
|
||||
|
||||
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
|
||||
|
||||
|
||||
def recompute_w_u_fwd(
|
||||
@@ -732,13 +655,13 @@ def chunk_gla_fwd_kernel_o(
|
||||
BV: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_tg = i_t
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -754,70 +677,44 @@ def chunk_gla_fwd_kernel_o(
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
o_i = tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
m_v = o_v < V
|
||||
m_tv = m_t[:, None] & m_v[None, :]
|
||||
m_A = m_t[:, None] & (o_i[None, :] < BT)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_h = tl.make_block_ptr(
|
||||
h + (i_tg * H + i_h) * K * V,
|
||||
(V, K),
|
||||
(K, 1),
|
||||
(i_v * BV, i_k * BK),
|
||||
(BV, BK),
|
||||
(1, 0),
|
||||
)
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_qk = m_t[:, None] & m_k[None, :]
|
||||
m_h = m_v[:, None] & m_k[None, :]
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
|
||||
|
||||
# [BT, BK]
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_q = (b_q * scale).to(b_q.dtype)
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
b_h = tl.load(p_h, mask=m_h, other=0.0)
|
||||
# [BT, BV]
|
||||
if i_k >= 0:
|
||||
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
|
||||
# [BT, BV]
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
b_v = tl.load(p_v, mask=m_tv, other=0.0)
|
||||
# [BT, BT]
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
|
||||
b_o += tl.dot(b_A, b_v, allow_tf32=False)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
|
||||
|
||||
|
||||
def chunk_gla_fwd_o_gk(
|
||||
@@ -896,12 +793,12 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
HAS_BIAS: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -911,26 +808,15 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
else:
|
||||
bos = i_b * T
|
||||
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * D,
|
||||
(T, D),
|
||||
(H * D, 1),
|
||||
(i_t * BT, i_d * BD),
|
||||
(BT, BD),
|
||||
(1, 0),
|
||||
)
|
||||
p_y = tl.make_block_ptr(
|
||||
y + (bos * H + i_h) * D,
|
||||
(T, D),
|
||||
(H * D, 1),
|
||||
(i_t * BT, i_d * BD),
|
||||
(BT, BD),
|
||||
(1, 0),
|
||||
)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_d = i_d * BD + tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_g = m_t[:, None] & (o_d[None, :] < D)
|
||||
p_g = g + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
p_y = y + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g = tl.load(p_g, mask=m_g, other=0.0).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
o_d = i_d * BD + tl.arange(0, BD)
|
||||
b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
@@ -951,12 +837,12 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
|
||||
# Out-of-bounds rows (load returns 0, but softplus/bias can still make
|
||||
# b_gate non-zero) participate in the dot product. They only contribute to
|
||||
# out-of-bounds output rows, which are masked away by `boundary_check` on
|
||||
# out-of-bounds output rows, which are masked away by `mask` on
|
||||
# the store, so visible output matches unfused gate + chunk-local cumsum.
|
||||
o_t = tl.arange(0, BT)
|
||||
m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)
|
||||
o_c = tl.arange(0, BT)
|
||||
m_cumsum = tl.where(o_c[:, None] >= o_c[None, :], 1.0, 0.0)
|
||||
b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale
|
||||
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_y, b_y.to(p_y.dtype.element_ty), mask=m_g)
|
||||
|
||||
|
||||
def fused_kda_gate_chunk_cumsum(
|
||||
@@ -1271,7 +1157,7 @@ def kda_gate_fwd_kernel(
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.load(A + i_h).to(tl.float32)
|
||||
@@ -1280,25 +1166,14 @@ def kda_gate_fwd_kernel(
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
|
||||
g_ptr = tl.make_block_ptr(
|
||||
base=g + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
o_t = n_t + tl.arange(0, BT)
|
||||
o_d = tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_g = m_t[:, None] & (o_d[None, :] < D)
|
||||
g_ptr = g + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
|
||||
y_ptr = y + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
|
||||
|
||||
y_ptr = tl.make_block_ptr(
|
||||
base=y + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
|
||||
|
||||
if HAS_BIAS:
|
||||
n_d = tl.arange(0, BD)
|
||||
@@ -1321,7 +1196,7 @@ def kda_gate_fwd_kernel(
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
|
||||
|
||||
|
||||
def fused_kda_gate(
|
||||
|
||||
@@ -396,16 +396,33 @@ def sparse_attn_indexer_kpool(
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
if rocm_aiter_ops.is_indexer_top_k_supported(
|
||||
is_prefill=True,
|
||||
compress_ratio=index_kpool,
|
||||
num_rows=num_rows,
|
||||
):
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_launch_aiter_top_k_per_row_prefill,
|
||||
)
|
||||
|
||||
_launch_aiter_top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
select_k,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
pool_ids = pool_topk.to(torch.int64)
|
||||
|
||||
@@ -33,9 +33,9 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, MLAAttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -89,15 +89,9 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
|
||||
The indexer shares one block with the co-located MLA (a single
|
||||
``MLAAttentionSpec`` / block_table), so ``block_size`` is the model-wide
|
||||
``cache_config.block_size``. DeepGEMM's paged-MQA kernel
|
||||
(``csrc/apis/attention.hpp``) requires ``block_kv`` to be exactly 32 or
|
||||
64, so the storage block is virtually split into pool pages of the
|
||||
largest such size that tiles it (``storage_kernel_block_size``); this
|
||||
needs ``block_size`` to be a multiple of ``index_kpool * 32`` (512 for
|
||||
``index_kpool = 16``). A smaller block (e.g. the default 64) silently
|
||||
collapses ``storage_block_size`` (64 // 16 = 4) and only fails later at
|
||||
the opaque C++ assert; ``get_kv_cache_spec`` guards this up front
|
||||
instead.
|
||||
``cache_config.block_size``. DeepGEMM's paged-MQA kernel takes 32- or
|
||||
64-state pages (only 64 on SM120), which ``Glm5NextIndexerBackend``
|
||||
declares as its kernel block sizes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -113,12 +107,6 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
|
||||
)
|
||||
assert index_kpool > 1, "Glm5NextIndexerCache expects index_kpool > 1"
|
||||
# Keep chunked-prefill boundaries aligned to complete pools.
|
||||
assert cache_config.block_size % index_kpool == 0, (
|
||||
"Glm5NextIndexerCache: cache_config.block_size "
|
||||
f"({cache_config.block_size}) must be a multiple of index_kpool "
|
||||
f"({index_kpool}) so chunked-prefill boundaries stay pool-aligned."
|
||||
)
|
||||
self._index_kpool = index_kpool
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig):
|
||||
@@ -128,33 +116,10 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
# ``tokens_per_state`` is the KV-spec representation of kpool
|
||||
# compression in the current cache-layout API.
|
||||
assert isinstance(spec, MLAAttentionSpec)
|
||||
spec = replace(spec, tokens_per_state=self._index_kpool)
|
||||
return replace(spec, tokens_per_state=self._index_kpool)
|
||||
|
||||
# DeepGEMM paged-MQA takes block_kv in {32, 64}; the storage block
|
||||
# (= block_size // index_kpool) is virtually split into pool pages of
|
||||
# the largest such size that tiles it, so it must be a multiple of 32.
|
||||
storage_block_size = spec.block_size // self._index_kpool
|
||||
assert (
|
||||
spec.block_size % self._index_kpool == 0 and storage_block_size % 32 == 0
|
||||
), (
|
||||
"Glm5NextIndexerCache: kpool indexer requires cache block_size to "
|
||||
f"be a multiple of index_kpool * 32 ({self._index_kpool * 32}) so "
|
||||
"that DeepGEMM paged-MQA pool pages (32 or 64 entries) tile the "
|
||||
f"storage block, got block_size={spec.block_size} -> "
|
||||
f"storage_block_size={storage_block_size}."
|
||||
)
|
||||
max_page_size = max(PAGED_MQA_PAGE_SIZES)
|
||||
min_page_size = min(PAGED_MQA_PAGE_SIZES)
|
||||
if storage_block_size <= max_page_size:
|
||||
page_size = storage_block_size
|
||||
elif storage_block_size % max_page_size == 0:
|
||||
page_size = max_page_size
|
||||
else:
|
||||
page_size = min_page_size
|
||||
return replace(
|
||||
spec,
|
||||
storage_block_size=page_size * self._index_kpool,
|
||||
)
|
||||
def get_attn_backend(self):
|
||||
return Glm5NextIndexerBackend
|
||||
|
||||
|
||||
class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
@@ -164,9 +129,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
``ring`` slots per request, overwritten in place by ``pos % ring``
|
||||
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
|
||||
tail raw K+gate); the connector transfers it across PD; decode reads it to
|
||||
compress the boundary pool correctly. ``KpoolTailSpec`` /
|
||||
``KpoolTailManager`` provide the no-prune, 1-block/req allocation that lets
|
||||
the in-progress pool survive across steps and across transfer.
|
||||
compress the boundary pool correctly. ``CircularBufferSpec`` provides the
|
||||
no-prune, 1-block/req allocation that lets the in-progress pool survive
|
||||
across steps and across transfer.
|
||||
|
||||
Stores raw bf16 K (``head_dim``) as the "K" half of each block and the
|
||||
bf16 gate score (``head_dim``) as the "V" half -- not the fp8-compressed
|
||||
@@ -202,13 +167,12 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
f"({self.cache_config.block_size}) must be a multiple of the "
|
||||
f"tail ring ({ring})"
|
||||
)
|
||||
return KpoolTailSpec(
|
||||
return CircularBufferSpec(
|
||||
block_size=ring,
|
||||
num_kv_heads=2,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=ring,
|
||||
# Tiny per-request ring; every DCP rank keeps the full tail so the
|
||||
# rank owning a pool can compress it.
|
||||
dcp_sharded=False,
|
||||
@@ -303,7 +267,7 @@ class Indexer(nn.Module):
|
||||
# Paged tail cache (in-progress pool's raw K + gate score). Written by
|
||||
# prefill (seeds the boundary pool) and decode (per-step stash); read by
|
||||
# the decode kernel to compress the boundary pool. Transferred across PD
|
||||
# so the decode side sees the prefill tail. See KpoolTailSpec/Manager.
|
||||
# so the decode side sees the prefill tail. See CircularBufferManager.
|
||||
self.tail_cache = Glm5NextTailCache(
|
||||
head_dim=self.head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
|
||||
@@ -181,7 +181,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
self.cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(
|
||||
|
||||
@@ -1092,7 +1092,9 @@ class Glm5NextForCausalLM(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
|
||||
vllm_config.model_config.dtype,
|
||||
vllm_config.cache_config.mamba_cache_dtype,
|
||||
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -15,6 +15,8 @@ from __future__ import annotations
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
|
||||
INDEX_HEAD_DIM = 128
|
||||
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
mask=do_write,
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
|
||||
if RETURN_COMPRESSED:
|
||||
tl.store(
|
||||
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
|
||||
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
mx_scale = write_cache and mqa_logits_use_mx_scales()
|
||||
# bytes per page (last dim of kv_cache) viewed as uint8
|
||||
buf_numel_per_page = buf.stride(0)
|
||||
s_offset_nbytes_in_page = page_size * head_dim
|
||||
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mx_scale,
|
||||
MX_SCALE=mx_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
WRITE_CACHE=write_cache,
|
||||
@@ -396,11 +409,6 @@ def _kpool_tail_seed_kernel(
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % RING, :]``.
|
||||
|
||||
The tail cache aliases the indexer cache with the indexer's (padded) block
|
||||
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
|
||||
``KPOOL_HEAD`` (``tail.stride(0)`` / ``tail.stride(1)``), never as a dense
|
||||
``[num_blocks, 2, KPOOL, HEAD_DIM]`` array.
|
||||
"""
|
||||
i = tl.program_id(0)
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
@@ -484,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per request; iterates its NEXT_N verify tokens in order.
|
||||
@@ -609,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
|
||||
# Stash the current token AFTER any completion read so the completion
|
||||
# uses prior stashes (and the current token's own key/score via
|
||||
@@ -723,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
|
||||
MX_SCALE=mqa_logits_use_mx_scales(),
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
+116
-241
@@ -230,13 +230,13 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
|
||||
NC: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_t, i_c, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
i_i, i_j = i_c // NC, i_c % NC
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -257,58 +257,51 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
|
||||
A += (bos * H + i_h) * BT
|
||||
Aqk += (bos * H + i_h) * BT
|
||||
|
||||
p_b = tl.make_block_ptr(
|
||||
beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)
|
||||
)
|
||||
b_b = tl.load(p_b, boundary_check=(0,))
|
||||
o_i = i_t * BT + i_i * BC + tl.arange(0, BC)
|
||||
o_j = i_t * BT + i_j * BC + tl.arange(0, BC)
|
||||
m_i = o_i < T
|
||||
m_j = o_j < T
|
||||
p_b = beta + bos * H + i_h + o_i * H
|
||||
b_b = tl.load(p_b, mask=m_i, other=0.0)
|
||||
|
||||
b_A = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
b_kt = tl.make_block_ptr(
|
||||
k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
|
||||
)
|
||||
p_gk = tl.make_block_ptr(
|
||||
g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
|
||||
)
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_qk = m_i[:, None] & m_k[None, :]
|
||||
m_kj = m_k[:, None] & m_j[None, :]
|
||||
|
||||
p_q = q + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + o_i[:, None] * (H * K) + o_k[None, :]
|
||||
b_kt = k + o_k[:, None] + o_j[None, :] * (H * K)
|
||||
p_gk = g + o_k[:, None] + o_j[None, :] * (H * K)
|
||||
|
||||
# [BK,]
|
||||
b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
|
||||
# [BC, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
b_k = tl.load(p_k, mask=m_qk, other=0.0) * exp2(b_g - b_gn[None, :])
|
||||
# [BK, BC]
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kt = tl.load(b_kt, boundary_check=(0, 1))
|
||||
b_gk = tl.load(p_gk, mask=m_kj, other=0.0)
|
||||
b_kt = tl.load(b_kt, mask=m_kj, other=0.0)
|
||||
# [BC, BC]
|
||||
b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
|
||||
b_A += tl.dot(b_k, b_ktg)
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
|
||||
b_Aqk += tl.dot(b_qg, b_ktg)
|
||||
|
||||
b_A *= b_b[:, None]
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_Aqk = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
o_jA = i_j * BC + tl.arange(0, BC)
|
||||
m_A = m_i[:, None] & (o_jA[None, :] < BT)
|
||||
p_A = A + o_i[:, None] * (H * BT) + o_jA[None, :]
|
||||
tl.store(p_A, b_A.to(A.dtype.element_ty), mask=m_A)
|
||||
p_Aqk = Aqk + o_i[:, None] * (H * BT) + o_jA[None, :]
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_A)
|
||||
|
||||
|
||||
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
|
||||
@@ -335,12 +328,12 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -359,33 +352,14 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
|
||||
m_A = (i_t * BT + i_i * BC + o_i) < T
|
||||
o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
|
||||
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT + i_i * BC, 0),
|
||||
(BC, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
o_c = i_t * BT + i_i * BC + tl.arange(0, BC)
|
||||
m_qk = m_A[:, None] & m_k[None, :]
|
||||
p_q = q + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + (bos * H + i_h) * K + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_k = tl.load(p_k, mask=m_qk, other=0.0)
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
|
||||
p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
|
||||
b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]
|
||||
@@ -538,12 +512,12 @@ def recompute_w_u_fwd_kernel(
|
||||
IS_VARLEN: tl.constexpr,
|
||||
DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -552,108 +526,57 @@ def recompute_w_u_fwd_kernel(
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,))
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
p_b = beta + bos * H + i_h + o_t * H
|
||||
b_b = tl.load(p_b, mask=m_t, other=0.0)
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
o_A = tl.arange(0, BT)
|
||||
m_A = m_t[:, None] & (o_A[None, :] < BT)
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
|
||||
for i_v in range(tl.cdiv(V, BV)):
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_u = tl.make_block_ptr(
|
||||
u + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
m_v = m_t[:, None] & (o_v[None, :] < V)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
b_v = tl.load(p_v, mask=m_v, other=0.0)
|
||||
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
|
||||
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
|
||||
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_w = tl.make_block_ptr(
|
||||
w + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_tk = m_t[:, None] & m_k[None, :]
|
||||
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_k = tl.load(p_k, mask=m_tk, other=0.0)
|
||||
b_kb = b_k * b_b[:, None]
|
||||
|
||||
p_gk = tl.make_block_ptr(
|
||||
gk + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_qg = tl.make_block_ptr(
|
||||
qg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_q = tl.load(p_q, mask=m_tk, other=0.0)
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
|
||||
|
||||
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
|
||||
|
||||
|
||||
def recompute_w_u_fwd(
|
||||
@@ -734,13 +657,13 @@ def chunk_gla_fwd_kernel_o(
|
||||
BV: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_tg = i_t
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -756,70 +679,44 @@ def chunk_gla_fwd_kernel_o(
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
o_i = tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
m_v = o_v < V
|
||||
m_tv = m_t[:, None] & m_v[None, :]
|
||||
m_A = m_t[:, None] & (o_i[None, :] < BT)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_h = tl.make_block_ptr(
|
||||
h + (i_tg * H + i_h) * K * V,
|
||||
(V, K),
|
||||
(K, 1),
|
||||
(i_v * BV, i_k * BK),
|
||||
(BV, BK),
|
||||
(1, 0),
|
||||
)
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_qk = m_t[:, None] & m_k[None, :]
|
||||
m_h = m_v[:, None] & m_k[None, :]
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
|
||||
|
||||
# [BT, BK]
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_q = (b_q * scale).to(b_q.dtype)
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
b_h = tl.load(p_h, mask=m_h, other=0.0)
|
||||
# [BT, BV]
|
||||
if i_k >= 0:
|
||||
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
|
||||
# [BT, BV]
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
b_v = tl.load(p_v, mask=m_tv, other=0.0)
|
||||
# [BT, BT]
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
|
||||
b_o += tl.dot(b_A, b_v, allow_tf32=False)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
|
||||
|
||||
|
||||
def chunk_gla_fwd_o_gk(
|
||||
@@ -898,12 +795,12 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
HAS_BIAS: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -913,26 +810,15 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
else:
|
||||
bos = i_b * T
|
||||
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * D,
|
||||
(T, D),
|
||||
(H * D, 1),
|
||||
(i_t * BT, i_d * BD),
|
||||
(BT, BD),
|
||||
(1, 0),
|
||||
)
|
||||
p_y = tl.make_block_ptr(
|
||||
y + (bos * H + i_h) * D,
|
||||
(T, D),
|
||||
(H * D, 1),
|
||||
(i_t * BT, i_d * BD),
|
||||
(BT, BD),
|
||||
(1, 0),
|
||||
)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_d = i_d * BD + tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_g = m_t[:, None] & (o_d[None, :] < D)
|
||||
p_g = g + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
p_y = y + (bos * H + i_h) * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g = tl.load(p_g, mask=m_g, other=0.0).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
o_d = i_d * BD + tl.arange(0, BD)
|
||||
b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
@@ -953,12 +839,12 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
|
||||
# Out-of-bounds rows (load returns 0, but softplus/bias can still make
|
||||
# b_gate non-zero) participate in the dot product. They only contribute to
|
||||
# out-of-bounds output rows, which are masked away by `boundary_check` on
|
||||
# out-of-bounds output rows, which are masked away by `mask` on
|
||||
# the store, so visible output matches unfused gate + chunk-local cumsum.
|
||||
o_t = tl.arange(0, BT)
|
||||
m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)
|
||||
o_c = tl.arange(0, BT)
|
||||
m_cumsum = tl.where(o_c[:, None] >= o_c[None, :], 1.0, 0.0)
|
||||
b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale
|
||||
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_y, b_y.to(p_y.dtype.element_ty), mask=m_g)
|
||||
|
||||
|
||||
def fused_kda_gate_chunk_cumsum(
|
||||
@@ -1273,7 +1159,7 @@ def kda_gate_fwd_kernel(
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.load(A + i_h).to(tl.float32)
|
||||
@@ -1282,25 +1168,14 @@ def kda_gate_fwd_kernel(
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
|
||||
g_ptr = tl.make_block_ptr(
|
||||
base=g + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
o_t = n_t + tl.arange(0, BT)
|
||||
o_d = tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_g = m_t[:, None] & (o_d[None, :] < D)
|
||||
g_ptr = g + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
|
||||
y_ptr = y + i_h * D + o_t[:, None] * stride_row + o_d[None, :] * stride_col
|
||||
|
||||
y_ptr = tl.make_block_ptr(
|
||||
base=y + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
|
||||
|
||||
if HAS_BIAS:
|
||||
n_d = tl.arange(0, BD)
|
||||
@@ -1323,7 +1198,7 @@ def kda_gate_fwd_kernel(
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
|
||||
|
||||
|
||||
def fused_kda_gate(
|
||||
|
||||
@@ -12,7 +12,12 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
PrefillRowShard,
|
||||
_merge_dcp_topk_global,
|
||||
_reserve_row_shard_exchange_transient,
|
||||
as_fp32_logits,
|
||||
)
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
_build_decode_scatter_indices,
|
||||
@@ -25,7 +30,13 @@ from vllm.models.glm5next.common.sparse_indexer import (
|
||||
)
|
||||
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
@@ -169,6 +180,10 @@ def sparse_attn_indexer_kpool(
|
||||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||||
)
|
||||
|
||||
# The dummy run returns before exchange_topk executes, so reserve its
|
||||
# worst-case transient for the memory profile.
|
||||
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
|
||||
|
||||
return topk_indices_buffer
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
|
||||
@@ -245,7 +260,7 @@ def sparse_attn_indexer_kpool(
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
@@ -282,6 +297,12 @@ def sparse_attn_indexer_kpool(
|
||||
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
|
||||
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
|
||||
_fill_causal_indices(_buf, _pos)
|
||||
# Short prefills score nothing locally; suppress the shard there.
|
||||
row_shard = None
|
||||
if not short_prefill:
|
||||
row_shard = PrefillRowShard.from_metadata(
|
||||
prefill_metadata, num_decode_tokens
|
||||
)
|
||||
|
||||
# Get the full shared workspace buffers once (will allocate on first use).
|
||||
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
|
||||
@@ -310,12 +331,16 @@ def sparse_attn_indexer_kpool(
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
row_start, row_end = chunk.token_start, chunk.token_end
|
||||
cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke
|
||||
if row_shard is not None:
|
||||
narrowed = row_shard.narrow(chunk)
|
||||
if narrowed is None:
|
||||
continue
|
||||
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
|
||||
|
||||
q_slice = q_quant[row_start:row_end]
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
@@ -325,20 +350,29 @@ def sparse_attn_indexer_kpool(
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
|
||||
k_scale_cast = k_scale.view(
|
||||
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
|
||||
).squeeze(-1)
|
||||
|
||||
num_rows = q_slice.shape[0]
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
|
||||
q_slice_cast, q_scale_slice, weights[row_start:row_end]
|
||||
)
|
||||
# Compressed: row i starts at K position cu_seqlen_ks[i].
|
||||
row_ends = cu_seqlen_ke - cu_seqlen_ks
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
chunk_weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
chunk.max_seqlen_k,
|
||||
),
|
||||
row_ends,
|
||||
)
|
||||
|
||||
# kpool: logits are pool-granular (compress_ratio == index_kpool),
|
||||
@@ -351,17 +385,15 @@ def sparse_attn_indexer_kpool(
|
||||
)
|
||||
topk_dst = pool_topk
|
||||
else:
|
||||
topk_dst = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
if logits.shape[1] == 0:
|
||||
topk_dst.fill_(-1)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
torch.zeros_like(row_ends),
|
||||
row_ends,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -376,7 +408,7 @@ def sparse_attn_indexer_kpool(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
pool_cp_interleave,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
row_starts=None,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
@@ -385,10 +417,7 @@ def sparse_attn_indexer_kpool(
|
||||
# Fused expand-pools + append-tail into one Triton kernel
|
||||
# (replaces ~25 elementwise ops). seq_len is token-granular
|
||||
# (pos+1); the kernel derives pool_len internally.
|
||||
q_seq = (
|
||||
positions[chunk.token_start : chunk.token_end].to(torch.int32)
|
||||
+ 1
|
||||
)
|
||||
q_seq = positions[row_start:row_end].to(torch.int32) + 1
|
||||
expanded = kpool_ops.expand_pools_and_append_tail(
|
||||
pool_ids, q_seq, index_kpool
|
||||
)
|
||||
@@ -397,9 +426,13 @@ def sparse_attn_indexer_kpool(
|
||||
expanded = kpool_ops.expand_pools_to_tokens(
|
||||
pool_ids, valid, topk_tokens, index_kpool
|
||||
)
|
||||
topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, : expanded.shape[-1]
|
||||
] = expanded
|
||||
topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded
|
||||
|
||||
if row_shard is not None:
|
||||
# The k-pool expansion appends the request's incomplete tail
|
||||
# (index_kpool - 1 entries) after the logical top-k history; those
|
||||
# columns are part of the attention index and are exchanged too.
|
||||
row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
@@ -577,18 +610,22 @@ def sparse_attn_indexer_kpool(
|
||||
if use_fp4_cache
|
||||
else padded_q_quant_decode_tokens
|
||||
)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
|
||||
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_pool_len,
|
||||
clean_logits=False,
|
||||
indices=decode_metadata.indices,
|
||||
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
|
||||
padded_q_quant_cast, padded_q_scale, padded_weights[:num_padded_tokens]
|
||||
)
|
||||
# SM100 decode is varlen: one row and context length per token.
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_pool_len,
|
||||
indices=decode_metadata.indices,
|
||||
),
|
||||
seq_lens.view(-1),
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
|
||||
|
||||
@@ -36,8 +36,6 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
@@ -435,12 +433,6 @@ class HYV4Model(nn.Module):
|
||||
split_expert_params_mapping = self.get_expert_mapping()
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Sink weights are sharded like the q/k/v linears.
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
@@ -572,48 +564,36 @@ class HYV4Model(nn.Module):
|
||||
if loaded_expert_param_names:
|
||||
loaded_params.update(loaded_expert_param_names)
|
||||
continue
|
||||
if "learnable_sink_param" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace(
|
||||
"gate.e_score_correction_bias", "expert_bias"
|
||||
)
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace("gate.e_score_correction_bias", "expert_bias")
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
@@ -493,13 +494,11 @@ class HYV4MLAAttention(nn.Module):
|
||||
if self.learnable_sink:
|
||||
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
|
||||
enable_sink = sink_backend is not None
|
||||
self.learnable_sink_param = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_local_heads,
|
||||
# The kernels require fp32 sinks; the disabled path keeps
|
||||
# the checkpoint dtype since the value is never consumed.
|
||||
dtype=torch.float32 if enable_sink else torch.bfloat16,
|
||||
)
|
||||
# The kernels require fp32 sinks; the disabled path keeps the
|
||||
# checkpoint dtype since the value is never consumed.
|
||||
self.learnable_sink_param = make_attention_sink(
|
||||
self.num_local_heads,
|
||||
torch.float32 if enable_sink else torch.bfloat16,
|
||||
)
|
||||
if enable_sink:
|
||||
sinks = self.learnable_sink_param
|
||||
|
||||
@@ -35,8 +35,6 @@ from transformers import PreTrainedConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
@@ -433,12 +431,6 @@ class HYV4Model(nn.Module):
|
||||
split_expert_params_mapping = self.get_expert_mapping()
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Sink weights are sharded like the q/k/v linears.
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
@@ -570,48 +562,36 @@ class HYV4Model(nn.Module):
|
||||
if loaded_expert_param_names:
|
||||
loaded_params.update(loaded_expert_param_names)
|
||||
continue
|
||||
if "learnable_sink_param" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace(
|
||||
"gate.e_score_correction_bias", "expert_bias"
|
||||
)
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace("gate.e_score_correction_bias", "expert_bias")
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
@@ -11,10 +11,6 @@ from torch import nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -744,11 +740,6 @@ class HYV4MTP(nn.Module):
|
||||
}
|
||||
|
||||
num_experts = getattr(self.config, "n_routed_experts", 0)
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
|
||||
# Routed-expert weights come in two checkpoint layouts:
|
||||
# split: mlp.experts.<id>.gate_proj.weight
|
||||
@@ -883,15 +874,6 @@ class HYV4MTP(nn.Module):
|
||||
if is_loaded:
|
||||
continue
|
||||
|
||||
if "learnable_sink_param" in name:
|
||||
if name in params_dict:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
|
||||
@@ -165,15 +165,23 @@ def _inkling_rel_attention_kernel(
|
||||
q_head_start = pid_kh * gqa_group_size
|
||||
bt_row = block_table_ptr + pid_b * stride_bt_b
|
||||
|
||||
q_block_ptr = tl.make_block_ptr(
|
||||
base=q_ptr + q_start * stride_q_t + q_head_start * stride_q_h,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_q_t, stride_q_h, stride_q_d),
|
||||
offsets=(q_block, 0, 0),
|
||||
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
|
||||
order=(2, 1, 0),
|
||||
o0 = q_block + tl.arange(0, BLOCK_Q)
|
||||
o1 = tl.arange(0, BLOCK_H)
|
||||
o2 = tl.arange(0, BLOCK_D)
|
||||
m_q = (
|
||||
(o0 < q_len)[:, None, None]
|
||||
& (o1 < gqa_group_size)[None, :, None]
|
||||
& (o2 < head_dim)[None, None, :]
|
||||
)
|
||||
q = tl.load(q_block_ptr, boundary_check=(0, 1, 2), padding_option="zero")
|
||||
q_block_ptr = (
|
||||
q_ptr
|
||||
+ q_start * stride_q_t
|
||||
+ q_head_start * stride_q_h
|
||||
+ o0[:, None, None] * stride_q_t
|
||||
+ o1[None, :, None] * stride_q_h
|
||||
+ o2[None, None, :] * stride_q_d
|
||||
)
|
||||
q = tl.load(q_block_ptr, mask=m_q, other=0.0)
|
||||
q = tl.reshape(q, (BLOCK_QH, BLOCK_D))
|
||||
|
||||
q_rows = q_block + tl.arange(0, BLOCK_Q)
|
||||
@@ -265,18 +273,18 @@ def _inkling_rel_attention_kernel(
|
||||
|
||||
acc /= l_i[:, None]
|
||||
acc = tl.reshape(acc, (BLOCK_Q, BLOCK_H, BLOCK_D))
|
||||
out_block_ptr = tl.make_block_ptr(
|
||||
base=out_ptr + q_start * stride_o_t + q_head_start * stride_o_h,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_o_t, stride_o_h, stride_o_d),
|
||||
offsets=(q_block, 0, 0),
|
||||
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
|
||||
order=(2, 1, 0),
|
||||
out_block_ptr = (
|
||||
out_ptr
|
||||
+ q_start * stride_o_t
|
||||
+ q_head_start * stride_o_h
|
||||
+ o0[:, None, None] * stride_o_t
|
||||
+ o1[None, :, None] * stride_o_h
|
||||
+ o2[None, None, :] * stride_o_d
|
||||
)
|
||||
tl.store(
|
||||
out_block_ptr,
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
boundary_check=(0, 1, 2),
|
||||
mask=m_q,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter,
|
||||
@@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_decode import (
|
||||
is_fused_kda_decode_supported,
|
||||
make_decode_conv1d_weight_loader,
|
||||
make_decode_norm_weight_loader,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
@@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.register_buffer(
|
||||
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
|
||||
)
|
||||
if decode_conv1d_weight is None:
|
||||
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
else:
|
||||
conv1d_weight_loader = make_decode_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
decode_conv1d_weight,
|
||||
)
|
||||
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader})
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
@@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
dtype=torch.float32,
|
||||
device=self.o_norm.weight.device,
|
||||
)
|
||||
if hasattr(self.o_norm.weight, "weight_loader"):
|
||||
delattr(self.o_norm.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.o_norm.weight,
|
||||
{"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)},
|
||||
)
|
||||
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
@@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@torch.no_grad()
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
if is_weights_pre_processed():
|
||||
return
|
||||
if self.decode_conv1d_weight is not None:
|
||||
self.decode_conv1d_weight.copy_(
|
||||
self.conv1d.weight.view(
|
||||
3, self.local_projection_size, self.conv_size
|
||||
).transpose(1, 2)
|
||||
)
|
||||
if self.decode_norm_weight is not None:
|
||||
self.decode_norm_weight.copy_(self.o_norm.weight)
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -50,8 +50,9 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
if attn.hisparse_cache is not None or attn.q_pad_num_heads is not None:
|
||||
return False
|
||||
# Only the W_UK variants reproduced in _fused_decode are supported.
|
||||
return attn.is_aiter_triton_fp4_bmm_enabled or (
|
||||
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
|
||||
return (
|
||||
attn.is_aiter_triton_fp4_bmm_enabled
|
||||
or not attn.is_aiter_triton_fp8_bmm_enabled
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
|
||||
|
||||
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
|
||||
The kernel in ``csrc/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
|
||||
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
|
||||
path with a single launch that keeps the per-chunk state in registers, so the
|
||||
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
|
||||
|
||||
@@ -2,22 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm entry points for the fused Kimi-K3 KDA decode kernel.
|
||||
|
||||
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_decode_kernel_rocm.cu``
|
||||
The kernel in ``csrc/kimi_k3/fused_kda_decode_kernel_rocm.cu``
|
||||
replaces, for a pure non-speculative decode batch, the three Triton launches
|
||||
and two copies the AMD KDA layer otherwise runs per layer: the packed causal
|
||||
conv1d update, the recurrent delta-rule step, and the gated output RMSNorm.
|
||||
|
||||
The kernel wants a width-major conv weight and an fp32 norm weight, so both are
|
||||
staged once at load time by the weight loaders below.
|
||||
The kernel wants a width-major conv weight and an fp32 norm weight, so the
|
||||
layer stages both once in ``process_weights_after_loading``.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -51,50 +48,3 @@ def is_fused_kda_decode_supported(
|
||||
# gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP /
|
||||
# bf16 primitives the kernel relies on.
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def make_decode_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
decode_conv1d_weight: torch.Tensor | None,
|
||||
) -> Callable[..., None]:
|
||||
"""Load the packed conv1d weight, mirroring a width-major fp32 copy.
|
||||
|
||||
The fused kernel indexes the weight as ``[qkv, width, channel]`` so the
|
||||
channel dimension is the contiguous one; the Triton prefill and fallback
|
||||
decode kernels keep the ``[channel, width]`` layout.
|
||||
"""
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
if decode_conv1d_weight is not None and not param.is_meta:
|
||||
decode_conv1d_weight[loaded_shard_id].copy_(
|
||||
loaded_shard.squeeze(1).transpose(0, 1)
|
||||
)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
def make_decode_norm_weight_loader(
|
||||
decode_norm_weight: torch.Tensor,
|
||||
) -> Callable[..., None]:
|
||||
"""Load the gated-norm weight, mirroring an fp32 copy for the kernel."""
|
||||
|
||||
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
default_weight_loader(param, loaded_weight)
|
||||
if not param.is_meta:
|
||||
decode_norm_weight.copy_(param.data)
|
||||
|
||||
return weight_loader
|
||||
|
||||
@@ -456,6 +456,7 @@ def _flashinfer_kda_prefill(
|
||||
beta_is_logit=True,
|
||||
seq_order=seq_order,
|
||||
prefill_workspace=prefill_workspace,
|
||||
backend="cute-dsl",
|
||||
)
|
||||
return output, initial_state
|
||||
|
||||
|
||||
@@ -95,10 +95,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4MegaMoEExperts,
|
||||
DeepseekV4MLP,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.nvidia.latent_moe_runner import (
|
||||
@@ -351,10 +348,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
self.activation = activation
|
||||
self.activation_beta = activation_beta
|
||||
self.activation_linear_beta = activation_linear_beta
|
||||
self.register_buffer("_mega_l1_packed", None, persistent=False)
|
||||
self.register_buffer("_mega_l1_scale", None, persistent=False)
|
||||
self.register_buffer("_mega_l2_packed", None, persistent=False)
|
||||
self.register_buffer("_mega_l2_scale", None, persistent=False)
|
||||
|
||||
def synchronize_first_launch(self) -> None:
|
||||
ep_group = get_ep_group()
|
||||
@@ -366,51 +359,11 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
torch.distributed.barrier(group=ep_group.cpu_group)
|
||||
self._synchronized_ep_groups.add(key)
|
||||
|
||||
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
|
||||
if self._transformed_l1_weights is not None:
|
||||
return
|
||||
|
||||
# Weight cache IPC engine: the daemon exported the transformed
|
||||
# buffers; reuse them zero-copy and drop the raw packed params.
|
||||
if self._mega_l1_packed is not None:
|
||||
self._transformed_l1_weights = (self._mega_l1_packed, self._mega_l1_scale)
|
||||
self._transformed_l2_weights = (self._mega_l2_packed, self._mega_l2_scale)
|
||||
self._drop_raw_mega_weights()
|
||||
return
|
||||
|
||||
self._check_runtime_supported()
|
||||
backend = self._ensure_backend()
|
||||
self._transformed_l1_weights, self._transformed_l2_weights = (
|
||||
backend.transform_weights(
|
||||
w13_weight=self.w13_weight.data,
|
||||
w13_weight_scale=self.w13_weight_scale.data,
|
||||
w2_weight=self.w2_weight.data,
|
||||
w2_weight_scale=self.w2_weight_scale.data,
|
||||
num_local_experts=self.num_local_experts,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
activation=self.activation,
|
||||
)
|
||||
)
|
||||
l1_packed, l1_scale = self._transformed_l1_weights
|
||||
l2_packed, l2_scale = self._transformed_l2_weights
|
||||
self.register_buffer("_mega_l1_packed", l1_packed, persistent=False)
|
||||
self.register_buffer("_mega_l1_scale", l1_scale, persistent=False)
|
||||
self.register_buffer("_mega_l2_packed", l2_packed, persistent=False)
|
||||
self.register_buffer("_mega_l2_scale", l2_scale, persistent=False)
|
||||
self._drop_raw_mega_weights()
|
||||
|
||||
def _drop_raw_mega_weights(self) -> None:
|
||||
self.w13_weight = None
|
||||
self.w13_weight_scale = None
|
||||
self.w2_weight = None
|
||||
self.w2_weight_scale = None
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
backend = self._ensure_backend()
|
||||
backend = self.backend
|
||||
group = get_ep_group().device_group
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (
|
||||
@@ -485,7 +438,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
else None,
|
||||
)
|
||||
|
||||
backend = self._ensure_backend()
|
||||
backend = self.backend
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
@@ -498,9 +451,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
is_padding=is_padding,
|
||||
hidden_quant=backend.hidden_quant,
|
||||
)
|
||||
self.finalize_weights()
|
||||
assert self._transformed_l1_weights is not None
|
||||
assert self._transformed_l2_weights is not None
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
@@ -566,6 +516,7 @@ class KimiMoE(nn.Module):
|
||||
)
|
||||
self.latent_moe_use_norm = config.latent_moe_use_norm
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.moe_renormalize = moe_renormalize
|
||||
self.use_grouped_topk = config.use_grouped_topk
|
||||
@@ -1572,11 +1523,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for module in self.modules():
|
||||
if isinstance(module, KimiMoE) and module.use_mega_moe:
|
||||
module.experts.finalize_weights()
|
||||
|
||||
|
||||
class KimiLinearForCausalLM(
|
||||
nn.Module,
|
||||
@@ -1699,14 +1645,6 @@ class KimiLinearForCausalLM(
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# A parent AutoWeightsLoader may invoke load_weights repeatedly for
|
||||
# non-contiguous streamed prefixes. Finalize only after the full stream.
|
||||
self.model.finalize_mega_moe_weights()
|
||||
# The fused MultiHeadLatentAttention's process_weights_after_loading
|
||||
# (W_UK_T / W_UV absorption) is driven by the loader's generic post-load
|
||||
# hook for any AttentionLayerBase, so no manual trigger is needed here.
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: KimiLinearConfig, weight_name: str
|
||||
@@ -2153,6 +2091,3 @@ class KimiK3ForConditionalGeneration(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.language_model.process_weights_after_loading()
|
||||
|
||||
@@ -421,11 +421,6 @@ class KimiK3MTP(nn.Module):
|
||||
f"MTP layer weights, or disable speculative decoding."
|
||||
)
|
||||
|
||||
if use_mega_moe:
|
||||
for module in self.modules():
|
||||
if isinstance(module, KimiMoE) and module.use_mega_moe:
|
||||
module.experts.finalize_weights()
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Fused MLA prefill and decode epilogues for Kimi-K3.
|
||||
|
||||
Thin wrappers over the CUDA ops in
|
||||
``csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu``, which
|
||||
``csrc/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu``, which
|
||||
mirror ``fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_{bf16,fp8}_insert``.
|
||||
|
||||
- ``fused_mla_key_concat_kv_cache_insert`` (bf16): optionally apply RoPE,
|
||||
|
||||
+75
-185
@@ -69,12 +69,12 @@ def recompute_w_u_fwd_kernel(
|
||||
IS_VARLEN: tl.constexpr,
|
||||
DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -83,108 +83,57 @@ def recompute_w_u_fwd_kernel(
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
p_b = beta + bos * H + i_h + o_t * H
|
||||
b_b = tl.load(p_b, mask=m_t, other=0.0).to(tl.float32)
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
o_A = tl.arange(0, BT)
|
||||
m_A = m_t[:, None] & (o_A[None, :] < BT)
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_A[None, :]
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
|
||||
for i_v in range(tl.cdiv(V, BV)):
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_u = tl.make_block_ptr(
|
||||
u + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
m_v = m_t[:, None] & (o_v[None, :] < V)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_u = u + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
b_v = tl.load(p_v, mask=m_v, other=0.0)
|
||||
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
|
||||
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), mask=m_v)
|
||||
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_w = tl.make_block_ptr(
|
||||
w + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_tk = m_t[:, None] & m_k[None, :]
|
||||
p_w = w + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_k = tl.load(p_k, mask=m_tk, other=0.0)
|
||||
b_kb = b_k * b_b[:, None]
|
||||
|
||||
p_gk = tl.make_block_ptr(
|
||||
gk + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
p_gk = gk + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_gk = tl.load(p_gk, mask=m_tk, other=0.0)
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_qg = tl.make_block_ptr(
|
||||
qg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_qg = qg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
b_q = tl.load(p_q, mask=m_tk, other=0.0)
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), mask=m_tk)
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
|
||||
p_kg = kg + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), mask=m_tk)
|
||||
|
||||
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), mask=m_tk)
|
||||
|
||||
|
||||
def recompute_w_u_fwd(
|
||||
@@ -265,13 +214,13 @@ def chunk_gla_fwd_kernel_o(
|
||||
BV: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_tg = i_t
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -287,70 +236,44 @@ def chunk_gla_fwd_kernel_o(
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
o_i = tl.arange(0, BT)
|
||||
m_t = o_t < T
|
||||
m_v = o_v < V
|
||||
m_tv = m_t[:, None] & m_v[None, :]
|
||||
m_A = m_t[:, None] & (o_i[None, :] < BT)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_h = tl.make_block_ptr(
|
||||
h + (i_tg * H + i_h) * K * V,
|
||||
(V, K),
|
||||
(K, 1),
|
||||
(i_v * BV, i_k * BK),
|
||||
(BV, BK),
|
||||
(1, 0),
|
||||
)
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_qk = m_t[:, None] & m_k[None, :]
|
||||
p_q = q + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + (bos * H + i_h) * K + o_t[:, None] * (H * K) + o_k[None, :]
|
||||
p_h = h + (i_tg * H + i_h) * K * V + o_v[:, None] * K + o_k[None, :]
|
||||
m_h = m_v[:, None] & m_k[None, :]
|
||||
|
||||
# [BT, BK]
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = tl.load(p_q, mask=m_qk, other=0.0)
|
||||
b_q = (b_q * scale).to(b_q.dtype)
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, mask=m_qk, other=0.0)
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
b_h = tl.load(p_h, mask=m_h, other=0.0)
|
||||
# [BT, BV]
|
||||
if i_k >= 0:
|
||||
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
p_v = v + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_o = o + (bos * H + i_h) * V + o_t[:, None] * (H * V) + o_v[None, :]
|
||||
p_A = A + (bos * H + i_h) * BT + o_t[:, None] * (H * BT) + o_i[None, :]
|
||||
# [BT, BV]
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
b_v = tl.load(p_v, mask=m_tv, other=0.0)
|
||||
# [BT, BT]
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
b_A = tl.load(p_A, mask=m_A, other=0.0)
|
||||
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
|
||||
b_o += tl.dot(b_A, b_v, allow_tf32=False)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_tv)
|
||||
|
||||
|
||||
def chunk_gla_fwd_o_gk(
|
||||
@@ -434,12 +357,12 @@ def kda_gate_chunk_cumsum_vector_kernel(
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1).to(tl.int64), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -472,34 +395,15 @@ def kda_gate_chunk_cumsum_vector_kernel(
|
||||
|
||||
i_s -= 1
|
||||
|
||||
p_s = tl.make_block_ptr(
|
||||
s + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_s = i_s * BS + tl.arange(0, BS)
|
||||
m_s = (o_t[:, None] < T) & (o_s[None, :] < S)
|
||||
p_s = s + (bos * H + i_h) * S + o_t[:, None] * (H * S) + o_s[None, :]
|
||||
p_o = o + (bos * H + i_h) * S + o_t[:, None] * (H * S) + o_s[None, :]
|
||||
|
||||
b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_s = tl.load(p_s, mask=m_s, other=0.0).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
p_bias = tl.make_block_ptr(
|
||||
g_bias + i_h * S,
|
||||
(S,),
|
||||
(1,),
|
||||
(i_s * BS,),
|
||||
(BS,),
|
||||
(0,),
|
||||
)
|
||||
b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32)
|
||||
b_bias = tl.load(g_bias + i_h * S + o_s, mask=o_s < S, other=0.0).to(tl.float32)
|
||||
b_s += b_bias[None, :]
|
||||
|
||||
b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
@@ -517,7 +421,7 @@ def kda_gate_chunk_cumsum_vector_kernel(
|
||||
# Boundary loads return zero, but bias and gate activation can make padded
|
||||
# rows nonzero. Padding trails valid rows, so it only affects masked stores.
|
||||
b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=m_s)
|
||||
|
||||
|
||||
def fused_kda_gate_chunk_cumsum(
|
||||
@@ -843,33 +747,19 @@ def kda_gate_fwd_kernel(
|
||||
HAS_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_h = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
o_t = n_t + tl.arange(0, BT)
|
||||
o_d = tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_g = m_t[:, None] & (o_d[None, :] < D)
|
||||
g_ptr = g + i_h * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
y_ptr = y + i_h * D + o_t[:, None] * (H * D) + o_d[None, :]
|
||||
|
||||
g_ptr = tl.make_block_ptr(
|
||||
base=g + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
y_ptr = tl.make_block_ptr(
|
||||
base=y + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g = tl.load(g_ptr, mask=m_g, other=0.0).to(tl.float32)
|
||||
|
||||
if HAS_BIAS:
|
||||
n_d = tl.arange(0, BD)
|
||||
@@ -887,7 +777,7 @@ def kda_gate_fwd_kernel(
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = -b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), mask=m_g)
|
||||
|
||||
|
||||
def fused_kda_gate(
|
||||
|
||||
+103
-167
@@ -70,14 +70,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
5. Computes merged Akk_inv
|
||||
6. Writes Akk_inv to Akk
|
||||
"""
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_t, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -106,6 +106,15 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
m_tc1 = (i_tc1 + o_i) < T
|
||||
m_tc2 = (i_tc2 + o_i) < T
|
||||
m_tc3 = (i_tc3 + o_i) < T
|
||||
o_c0 = i_tc0 + o_i
|
||||
o_c1 = i_tc1 + o_i
|
||||
o_c2 = i_tc2 + o_i
|
||||
o_c3 = i_tc3 + o_i
|
||||
m_tc0 = o_c0 < T
|
||||
m_A0 = m_tc0[:, None] & (o_i[None, :] < BT)
|
||||
m_A1 = m_tc1[:, None] & (o_i[None, :] < BT)
|
||||
m_A2 = m_tc2[:, None] & (o_i[None, :] < BT)
|
||||
m_A3 = m_tc3[:, None] & (o_i[None, :] < BT)
|
||||
|
||||
b_Aqk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
@@ -129,29 +138,21 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
|
||||
p_k0 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g0 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
|
||||
m_ck0 = m_tc0[:, None] & m_k[None, :]
|
||||
p_k0 = k + o_c0[:, None] * (H * K) + o_k[None, :]
|
||||
p_g0 = g + o_c0[:, None] * (HV * K) + o_k[None, :]
|
||||
b_k0 = tl.load(p_k0, mask=m_ck0, other=0.0).to(tl.float32)
|
||||
b_g0 = tl.load(p_g0, mask=m_ck0, other=0.0).to(tl.float32)
|
||||
|
||||
if i_tc1 < T:
|
||||
p_q1 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k1 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g1 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
m_ck1 = m_tc1[:, None] & m_k[None, :]
|
||||
p_q1 = q + o_c1[:, None] * (H * K) + o_k[None, :]
|
||||
p_k1 = k + o_c1[:, None] * (H * K) + o_k[None, :]
|
||||
p_g1 = g + o_c1[:, None] * (HV * K) + o_k[None, :]
|
||||
# [BC, BK]
|
||||
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_q1 = tl.load(p_q1, mask=m_ck1, other=0.0).to(tl.float32)
|
||||
b_k1 = tl.load(p_k1, mask=m_ck1, other=0.0).to(tl.float32)
|
||||
b_g1 = tl.load(p_g1, mask=m_ck1, other=0.0).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
@@ -163,19 +164,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
|
||||
|
||||
if i_tc2 < T:
|
||||
p_q2 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k2 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g2 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
m_ck2 = m_tc2[:, None] & m_k[None, :]
|
||||
p_q2 = q + o_c2[:, None] * (H * K) + o_k[None, :]
|
||||
p_k2 = k + o_c2[:, None] * (H * K) + o_k[None, :]
|
||||
p_g2 = g + o_c2[:, None] * (HV * K) + o_k[None, :]
|
||||
# [BC, BK]
|
||||
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_q2 = tl.load(p_q2, mask=m_ck2, other=0.0).to(tl.float32)
|
||||
b_k2 = tl.load(p_k2, mask=m_ck2, other=0.0).to(tl.float32)
|
||||
b_g2 = tl.load(p_g2, mask=m_ck2, other=0.0).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
@@ -195,19 +191,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
b_Akk21 += tl.dot(b_kg2, b_kgt)
|
||||
|
||||
if i_tc3 < T:
|
||||
p_q3 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k3 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g3 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
m_ck3 = m_tc3[:, None] & m_k[None, :]
|
||||
p_q3 = q + o_c3[:, None] * (H * K) + o_k[None, :]
|
||||
p_k3 = k + o_c3[:, None] * (H * K) + o_k[None, :]
|
||||
p_g3 = g + o_c3[:, None] * (HV * K) + o_k[None, :]
|
||||
# [BC, BK]
|
||||
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_q3 = tl.load(p_q3, mask=m_ck3, other=0.0).to(tl.float32)
|
||||
b_k3 = tl.load(p_k3, mask=m_ck3, other=0.0).to(tl.float32)
|
||||
b_g3 = tl.load(p_g3, mask=m_ck3, other=0.0).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
@@ -236,82 +227,44 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
# save off-diagonal Aqk blocks and prepare Akk
|
||||
################################################################################
|
||||
if i_tc1 < T:
|
||||
p_Aqk10 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
p_Aqk10 = Aqk + o_c1[:, None] * (HV * BT) + o_i[None, :]
|
||||
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), mask=m_A1)
|
||||
|
||||
p_b1 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
|
||||
)
|
||||
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
|
||||
p_b1 = beta + bos * HV + i_hv + o_c1 * HV
|
||||
b_b1 = tl.load(p_b1, mask=m_tc1, other=0.0).to(tl.float32)
|
||||
b_Akk10 = b_Akk10 * b_b1[:, None]
|
||||
if i_tc2 < T:
|
||||
p_Aqk20 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk21 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
p_Aqk20 = Aqk + o_c2[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Aqk21 = Aqk + o_c2[:, None] * (HV * BT) + (o_i + BC)[None, :]
|
||||
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), mask=m_A2)
|
||||
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), mask=m_A2)
|
||||
|
||||
p_b2 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
|
||||
)
|
||||
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
|
||||
p_b2 = beta + bos * HV + i_hv + o_c2 * HV
|
||||
b_b2 = tl.load(p_b2, mask=m_tc2, other=0.0).to(tl.float32)
|
||||
b_Akk20 = b_Akk20 * b_b2[:, None]
|
||||
b_Akk21 = b_Akk21 * b_b2[:, None]
|
||||
if i_tc3 < T:
|
||||
p_Aqk30 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk31 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk32 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
p_Aqk30 = Aqk + o_c3[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Aqk31 = Aqk + o_c3[:, None] * (HV * BT) + (o_i + BC)[None, :]
|
||||
p_Aqk32 = Aqk + o_c3[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
|
||||
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
|
||||
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
|
||||
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), mask=m_A3)
|
||||
|
||||
p_b3 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
|
||||
)
|
||||
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
|
||||
p_b3 = beta + bos * HV + i_hv + o_c3 * HV
|
||||
b_b3 = tl.load(p_b3, mask=m_tc3, other=0.0).to(tl.float32)
|
||||
b_Akk30 = b_Akk30 * b_b3[:, None]
|
||||
b_Akk31 = b_Akk31 * b_b3[:, None]
|
||||
b_Akk32 = b_Akk32 * b_b3[:, None]
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32)
|
||||
p_Akk00 = Akkd + o_c0[:, None] * (HV * BC) + o_i[None, :]
|
||||
p_Akk11 = Akkd + o_c1[:, None] * (HV * BC) + o_i[None, :]
|
||||
p_Akk22 = Akkd + o_c2[:, None] * (HV * BC) + o_i[None, :]
|
||||
p_Akk33 = Akkd + o_c3[:, None] * (HV * BC) + o_i[None, :]
|
||||
b_Ai00 = tl.load(p_Akk00, mask=m_A0, other=0.0).to(tl.float32)
|
||||
b_Ai11 = tl.load(p_Akk11, mask=m_A1, other=0.0).to(tl.float32)
|
||||
b_Ai22 = tl.load(p_Akk22, mask=m_A2, other=0.0).to(tl.float32)
|
||||
b_Ai33 = tl.load(p_Akk33, mask=m_A3, other=0.0).to(tl.float32)
|
||||
|
||||
################################################################################
|
||||
# forward substitution on diagonals
|
||||
@@ -397,47 +350,27 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
# store full Akk_inv to Akk
|
||||
################################################################################
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk10 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk20 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk21 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk30 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk31 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk32 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk00 = Akk + o_c0[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Akk10 = Akk + o_c1[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Akk11 = Akk + o_c1[:, None] * (HV * BT) + (o_i + BC)[None, :]
|
||||
p_Akk20 = Akk + o_c2[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Akk21 = Akk + o_c2[:, None] * (HV * BT) + (o_i + BC)[None, :]
|
||||
p_Akk22 = Akk + o_c2[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
|
||||
p_Akk30 = Akk + o_c3[:, None] * (HV * BT) + o_i[None, :]
|
||||
p_Akk31 = Akk + o_c3[:, None] * (HV * BT) + (o_i + BC)[None, :]
|
||||
p_Akk32 = Akk + o_c3[:, None] * (HV * BT) + (o_i + 2 * BC)[None, :]
|
||||
p_Akk33 = Akk + o_c3[:, None] * (HV * BT) + (o_i + 3 * BC)[None, :]
|
||||
|
||||
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), mask=m_A0)
|
||||
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), mask=m_A1)
|
||||
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), mask=m_A1)
|
||||
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), mask=m_A2)
|
||||
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), mask=m_A2)
|
||||
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), mask=m_A2)
|
||||
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), mask=m_A3)
|
||||
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), mask=m_A3)
|
||||
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), mask=m_A3)
|
||||
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), mask=m_A3)
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
@@ -474,14 +407,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_GATHER: tl.constexpr,
|
||||
):
|
||||
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_t, i_i, i_bh = tl.program_id(0).to(tl.int64), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int64),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
@@ -505,16 +438,19 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
Aqk = Aqk + (bos * HV + i_hv) * BT
|
||||
Akk = Akk + (bos * HV + i_hv) * BC
|
||||
|
||||
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
o_k = tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
m_ck = m_c[:, None] & m_k[None, :]
|
||||
p_q = q + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_k = k + o_c[:, None] * (H * K) + o_k[None, :]
|
||||
p_g = g + o_c[:, None] * (HV * K) + o_k[None, :]
|
||||
|
||||
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
|
||||
p_beta = beta + o_c * HV
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
b_q = tl.load(p_q, mask=m_ck, other=0.0)
|
||||
b_k = tl.load(p_k, mask=m_ck, other=0.0)
|
||||
b_g = tl.load(p_g, mask=m_ck, other=0.0)
|
||||
b_beta = tl.load(p_beta, mask=m_c, other=0.0).to(tl.float32)
|
||||
|
||||
if USE_GATHER:
|
||||
b_gn = gather(
|
||||
@@ -546,12 +482,12 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
|
||||
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
|
||||
|
||||
p_Aqk = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
m_Aqk_st = m_c[:, None] & (o_i[None, :] < BT)
|
||||
m_Akk_st = m_c[:, None] & (o_i[None, :] < BC)
|
||||
p_Aqk = Aqk + o_c[:, None] * (HV * BT) + (i_i * BC + o_i)[None, :]
|
||||
p_Akk = Akk + o_c[:, None] * (HV * BC) + o_i[None, :]
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), mask=m_Aqk_st)
|
||||
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), mask=m_Akk_st)
|
||||
|
||||
tl.debug_barrier()
|
||||
|
||||
@@ -566,7 +502,7 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
|
||||
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
|
||||
b_Ai += m_I
|
||||
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), mask=m_Akk_st)
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra(
|
||||
|
||||
+6
-10
@@ -108,20 +108,16 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
|
||||
# g: [B, T, HV, K], beta: [B, T, HV]
|
||||
p_g = tl.make_block_ptr(
|
||||
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
p_g = g + i_t * HV * K + o_hv[:, None] * K + o_k[None, :]
|
||||
p_beta = beta + i_t * HV + o_hv
|
||||
b_g = tl.load(p_g, mask=m_hk, other=0.0).to(tl.float32)
|
||||
b_beta = tl.load(p_beta, mask=m_hv, other=0.0).to(tl.float32)
|
||||
b_k *= b_beta[:, None]
|
||||
|
||||
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
|
||||
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
p_gj = tl.make_block_ptr(
|
||||
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
|
||||
p_gj = g + j * HV * K + o_hv[:, None] * K + o_k[None, :]
|
||||
b_gj = tl.load(p_gj, mask=m_hk, other=0.0).to(tl.float32)
|
||||
|
||||
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
|
||||
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
|
||||
|
||||
@@ -67,6 +67,8 @@ MSA_TOPK_BLOCKS = 16
|
||||
MSA_SCORE_TYPE = "max"
|
||||
MSA_SPARSE_BLOCK_SIZE = 128
|
||||
MSA_INDEX_HEAD_DIM = 128
|
||||
# Index head counts the score kernels are instantiated for.
|
||||
MSA_NUM_INDEX_HEADS = (1, 2)
|
||||
# Wave width the top-k is written against: it gives one lane per output slot and
|
||||
# reads the score row in wave-wide strips.
|
||||
WAVE_SIZE = 64
|
||||
@@ -181,8 +183,11 @@ def aiter_indexer_unsupported_reason(
|
||||
f"needs index_head_dim={MSA_INDEX_HEAD_DIM}, "
|
||||
f"got index_head_dim={index_head_dim}"
|
||||
)
|
||||
if num_index_heads > MFMA_COLS:
|
||||
return f"num_index_heads={num_index_heads} exceeds the {MFMA_COLS} MFMA columns"
|
||||
if num_index_heads not in MSA_NUM_INDEX_HEADS:
|
||||
return (
|
||||
f"needs num_index_heads in {MSA_NUM_INDEX_HEADS}, "
|
||||
f"got num_index_heads={num_index_heads}"
|
||||
)
|
||||
# A decode row's whole query shares one MFMA tile, one column per (token,
|
||||
# head) pair, so spec decode trades columns against the head count.
|
||||
if num_index_heads * max_decode_query_len > MFMA_COLS:
|
||||
|
||||
@@ -115,15 +115,17 @@ def _index_block_score_kernel(
|
||||
if BLOCK_SIZE_Q * pid_q >= q_len:
|
||||
return
|
||||
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
|
||||
shape=(q_len, head_dim),
|
||||
strides=(stride_q_n, stride_q_d),
|
||||
offsets=(pid_q * BLOCK_SIZE_Q, 0),
|
||||
block_shape=(BLOCK_SIZE_Q, head_dim),
|
||||
order=(1, 0),
|
||||
o_q = pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)
|
||||
o_d = tl.arange(0, head_dim)
|
||||
m_q = o_q < q_len
|
||||
q_ptrs = (
|
||||
q_ptr
|
||||
+ seq_start.to(tl.int64) * stride_q_n
|
||||
+ pid_h.to(tl.int64) * stride_q_h
|
||||
+ o_q[:, None].to(tl.int64) * stride_q_n
|
||||
+ o_d[None, :].to(tl.int64) * stride_q_d
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
|
||||
q = tl.load(q_ptrs, mask=m_q[:, None], other=0.0)
|
||||
# A matched fp8 pair (e.g. e4m3 x e4m3) lowers to a native FP8 MMA, so
|
||||
# upcast only when the operands differ: there is no mixed-dtype fp8 MMA,
|
||||
# including across fp8 flavours (fp8e4nv vs fp8e4b8). bf16 and fp32 loads
|
||||
|
||||
@@ -125,7 +125,9 @@ def minimax_m3_use_aiter_sparse_pa(
|
||||
raise ValueError(
|
||||
"MiniMax M3 AITER sparse paged attention requires "
|
||||
f"num_kv_heads == 1 per tensor-parallel rank, got {num_kv_heads}, "
|
||||
"unless the indexer emits the attend's page table."
|
||||
"unless the AITER indexer emits the attend's page table. It is not "
|
||||
"in use here; the 'MiniMax M3 indexer: AITER unavailable' log line "
|
||||
"says why."
|
||||
)
|
||||
return requested
|
||||
|
||||
|
||||
@@ -614,7 +614,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
|
||||
q = qkv.new_empty((num_tokens, self.q_size))
|
||||
# With query_fp8, q is emitted only in fp8; the attend dequantizes it
|
||||
# into ``q`` for the tokens that run on bf16-query kernels.
|
||||
# into ``q`` for the tokens that run on Triton decode.
|
||||
query_fp8 = self._allocate_query_fp8(qkv)
|
||||
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
|
||||
# the fused kernel emits fp8 directly when this buffer is e4m3.
|
||||
|
||||
@@ -23,9 +23,10 @@ _TOPK = 16
|
||||
# fixed planner allocation used by the MSA decode kernel.
|
||||
_MAX_QUERY_HEAD_ROWS = 65536
|
||||
_MAX_DECODE_QUERY_LEN = 32
|
||||
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
|
||||
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
|
||||
_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
_DEFAULT_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
_B300_EAGLE3_QUERY_HEAD_ROWS = 1024
|
||||
_B300_EAGLE3_MAX_QUERY_LEN = 8
|
||||
_B300_EAGLE3_HEAD_GEOMETRIES = {(64, 4), (32, 2), (16, 1)}
|
||||
|
||||
|
||||
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
@@ -235,6 +236,22 @@ def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _min_cutlass_batch_size(
|
||||
decode_query_len: int,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> int:
|
||||
if (
|
||||
not current_platform.is_device_capability((10, 3))
|
||||
or not 1 <= decode_query_len <= _B300_EAGLE3_MAX_QUERY_LEN
|
||||
or (num_q_heads, num_kv_heads) not in _B300_EAGLE3_HEAD_GEOMETRIES
|
||||
):
|
||||
return _DEFAULT_MIN_CUTLASS_BATCH_SIZE
|
||||
# Use CUTLASS once batch_size * decode_query_len * num_q_heads >= 1024.
|
||||
query_head_rows = decode_query_len * num_q_heads
|
||||
return (_B300_EAGLE3_QUERY_HEAD_ROWS + query_head_rows - 1) // query_head_rows
|
||||
|
||||
|
||||
def supports_cutlass_sparse_decode(
|
||||
*,
|
||||
decode_backend: MiniMaxM3MSADecodeBackend,
|
||||
@@ -267,9 +284,14 @@ def should_prepare_decode_metadata(
|
||||
kv_cache_dtype: str,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
min_batch_size: int | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a graph shape can use the CUTLASS decode path."""
|
||||
total_q = batch_size * decode_query_len
|
||||
if min_batch_size is None:
|
||||
min_batch_size = _min_cutlass_batch_size(
|
||||
decode_query_len, num_q_heads, num_kv_heads
|
||||
)
|
||||
return (
|
||||
supports_cutlass_sparse_decode(
|
||||
decode_backend=decode_backend,
|
||||
@@ -280,7 +302,7 @@ def should_prepare_decode_metadata(
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
|
||||
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
|
||||
and (batch_size >= min_batch_size or is_nvfp4_kv_cache(kv_cache_dtype))
|
||||
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
|
||||
)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ path for regular decode and speculative verification. NVFP4 KV caches read the
|
||||
vLLM packed pages directly in both prefill and CUTLASS decode.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -49,6 +50,11 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_msa_cutlass_min_batch_size() -> int | None:
|
||||
value = os.getenv("VLLM_MINIMAX_M3_MSA_CUTLASS_MIN_BATCH")
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
def _dequantize_query(
|
||||
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
@@ -133,6 +139,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
|
||||
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
|
||||
self.msa_cutlass_min_batch_size = _get_msa_cutlass_min_batch_size()
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -159,6 +166,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
page_size=SPARSE_BLOCK_SIZE,
|
||||
topk_blocks=self.topk_blocks,
|
||||
min_batch_size=self.msa_cutlass_min_batch_size,
|
||||
):
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
@@ -266,7 +274,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
||||
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
||||
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
|
||||
# decode reads it directly, and the bf16-query kernels read it
|
||||
# decode and prefill read it directly, and Triton decode reads it
|
||||
# dequantized into ``query``.
|
||||
q_fp8 = (
|
||||
None
|
||||
@@ -339,11 +347,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
|
||||
# strided view directly (topK stays innermost-contiguous).
|
||||
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
||||
qp = (
|
||||
q[nd:]
|
||||
if q_fp8 is None
|
||||
else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:])
|
||||
)
|
||||
# An E4M3 query attends as is, its scale folded into softmax_scale.
|
||||
if q_fp8 is None:
|
||||
qp, softmax_scale = q[nd:], self.scale
|
||||
else:
|
||||
qp, softmax_scale = q_fp8[nd:], self.scale * q_scale_float
|
||||
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
||||
prefill_topk,
|
||||
p.cu_seqlens_q,
|
||||
@@ -375,7 +383,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
max_seqlen_k=p.max_seq_len,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
softmax_scale=softmax_scale,
|
||||
page_table=p.block_table,
|
||||
seqused_k=p.seq_lens,
|
||||
schedule=schedule,
|
||||
@@ -393,7 +401,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
topK=self.topk_blocks,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
softmax_scale=softmax_scale,
|
||||
cu_seqlens_q=p.cu_seqlens_q,
|
||||
cu_seqlens_k=p.cu_seqlens_k,
|
||||
max_seqlen_q=p.max_query_len,
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.v1.attention.backends.short_conv_attn import (
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
)
|
||||
from ..common.ops.ple import ple_conv, ple_gate, ple_ngram_ids
|
||||
@@ -241,6 +242,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
embedding_prefix,
|
||||
getattr(config, "ple_embedding_dtype", None),
|
||||
)
|
||||
if isinstance(embedding_quant_method, Qwen4ExpPLENvFp4EmbeddingMethod):
|
||||
raise NotImplementedError(
|
||||
"NVFP4 Qwen4Exp PLE tables are not supported on ROCm yet; use a "
|
||||
"checkpoint with a BF16 or FP8 PLE table."
|
||||
)
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
embedding_cls = (
|
||||
Qwen4ExpPLEPinnedHostEmbedding
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLine
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
@@ -248,6 +249,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
head_size=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||
)
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -36,6 +36,11 @@ from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
_e2m1_inline,
|
||||
_e4m3_inline,
|
||||
dequantize_to_dtype,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
@@ -70,6 +75,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
del num_ngram_heads, max_total_tokens
|
||||
super().__init__(
|
||||
@@ -80,6 +86,7 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
prefix=prefix,
|
||||
quant_method=embedding_method,
|
||||
parallel_group=get_etp_group(),
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
self.embedding_method = embedding_method
|
||||
self.data_parallel_rank = data_parallel_rank
|
||||
@@ -153,14 +160,15 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
return embeddings.narrow(0, slot_offset, local_num_tokens)
|
||||
|
||||
@abstractmethod
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Start an asynchronous lookup when supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Return prefetched embeddings for the full input token set."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
"""Quantization interface shared by device, pinned, and file-gather PLE tables."""
|
||||
@@ -175,13 +183,28 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
embedding_dtype: str | None = None,
|
||||
) -> "Qwen4ExpPLEEmbeddingMethod":
|
||||
"""Select the concrete PLE embedding format for a layer."""
|
||||
if embedding_dtype == "nvfp4":
|
||||
return Qwen4ExpPLENvFp4EmbeddingMethod()
|
||||
if embedding_dtype == "float8_e4m3fn":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if embedding_dtype is not None:
|
||||
raise ValueError(
|
||||
"ple_embedding_dtype must be 'nvfp4' or 'float8_e4m3fn', "
|
||||
f"got {embedding_dtype!r}"
|
||||
)
|
||||
if quant_config is None:
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
if quant_config._resolve_quant_algo(prefix) == "FP8":
|
||||
quant_algo = quant_config._resolve_quant_algo(prefix)
|
||||
if quant_algo == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if quant_algo == "NVFP4":
|
||||
return Qwen4ExpPLENvFp4EmbeddingMethod()
|
||||
if quant_algo is not None:
|
||||
raise ValueError(
|
||||
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
|
||||
f"{prefix!r}; expected FP8 or NVFP4"
|
||||
)
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(
|
||||
quant_config, ModelOptQuantConfigBase
|
||||
@@ -235,6 +258,28 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||
return F.embedding(input_, layer.weight)
|
||||
|
||||
def lookup_dtype(self, layer: nn.Module) -> torch.dtype:
|
||||
"""Return the dtype of rows that a pinned-host lookup writes."""
|
||||
return layer.weight.dtype
|
||||
|
||||
def lookup_from_pinned(
|
||||
self,
|
||||
layer: "Qwen4ExpPLEPinnedHostEmbedding",
|
||||
ids: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy the ETP-owned rows for ``ids`` from pinned host memory."""
|
||||
row_bytes = layer.embedding_dim * layer.weight.element_size()
|
||||
_lookup_ple_embedding_from_pinned_kernel[(ids.numel(),)](
|
||||
layer._uva_weight,
|
||||
ids,
|
||||
output,
|
||||
row_bytes,
|
||||
layer.shard_indices.org_vocab_start_index,
|
||||
layer.shard_indices.org_vocab_end_index,
|
||||
BLOCK_D=triton.next_power_of_2(row_bytes),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
"""Bind host-file-gather tables once every checkpoint shard is loaded."""
|
||||
bind = getattr(layer, "bind_file_shards", None)
|
||||
@@ -346,6 +391,178 @@ class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
|
||||
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||
|
||||
|
||||
# NVFP4 packs two E2M1 values per byte and shares one E4M3 scale per block.
|
||||
_NVFP4_VALUES_PER_BYTE = 2
|
||||
_NVFP4_BLOCK_SIZE = 16
|
||||
|
||||
|
||||
class Qwen4ExpPLENvFp4EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
|
||||
"""NVFP4 PLE embedding: packed E2M1 rows, block scales, one global scale.
|
||||
|
||||
Rows stay packed in storage. A lookup decodes only the selected rows to
|
||||
the activation dtype, before the ETP reduction.
|
||||
"""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Qwen4ExpPLEEmbedding,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size, params_dtype
|
||||
if input_size_per_partition % _NVFP4_BLOCK_SIZE:
|
||||
raise ValueError(
|
||||
"NVFP4 PLE embedding dimension must be divisible by "
|
||||
f"{_NVFP4_BLOCK_SIZE}, got {input_size_per_partition}"
|
||||
)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
num_rows = sum(output_partition_sizes)
|
||||
for name, width, dtype in (
|
||||
(
|
||||
"weight",
|
||||
input_size_per_partition // _NVFP4_VALUES_PER_BYTE,
|
||||
torch.uint8,
|
||||
),
|
||||
(
|
||||
"weight_scale",
|
||||
input_size_per_partition // _NVFP4_BLOCK_SIZE,
|
||||
torch.float8_e4m3fn,
|
||||
),
|
||||
):
|
||||
parameter = ModelWeightParameter(
|
||||
data=layer.allocate_embedding_weight(num_rows, width, dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter(name, parameter)
|
||||
weight_scale_2 = create_fp8_scale_parameter(
|
||||
PerTensorScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
None,
|
||||
weight_loader,
|
||||
scale_dtype=torch.float32,
|
||||
)
|
||||
layer.register_parameter("weight_scale_2", weight_scale_2)
|
||||
if layer.weight_scale.is_pinned():
|
||||
self._uva_weight_scale = get_accelerator_view_from_cpu_tensor(
|
||||
layer.weight_scale
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
"""Reject NVFP4 PLE checkpoints without a global scale."""
|
||||
sentinel = torch.finfo(torch.float32).min
|
||||
if torch.any(layer.weight_scale_2 == sentinel):
|
||||
raise ValueError("NVFP4 PLE checkpoint is missing its global scale")
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
def lookup_dtype(self, layer: nn.Module) -> torch.dtype:
|
||||
return layer.params_dtype
|
||||
|
||||
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||
ids = input_.reshape(-1)
|
||||
if not input_.is_cuda:
|
||||
rows = dequantize_to_dtype(
|
||||
F.embedding(ids, layer.weight),
|
||||
F.embedding(ids, layer.weight_scale),
|
||||
layer.weight_scale_2,
|
||||
layer.params_dtype,
|
||||
swizzle=False,
|
||||
)
|
||||
return rows.reshape(*input_.shape, layer.embedding_dim)
|
||||
output = torch.empty(
|
||||
(*input_.shape, layer.embedding_dim),
|
||||
dtype=layer.params_dtype,
|
||||
device=input_.device,
|
||||
)
|
||||
if ids.numel():
|
||||
# Device rows are already local, so every ID in range is owned.
|
||||
_lookup_nvfp4_ple_embedding_kernel[(ids.numel(),)](
|
||||
layer.weight,
|
||||
layer.weight_scale.view(torch.uint8),
|
||||
layer.weight_scale_2,
|
||||
ids,
|
||||
output,
|
||||
layer.embedding_dim,
|
||||
0,
|
||||
layer.weight.shape[0],
|
||||
BLOCK_D=triton.next_power_of_2(layer.embedding_dim),
|
||||
)
|
||||
return output
|
||||
|
||||
def lookup_from_pinned(
|
||||
self,
|
||||
layer: "Qwen4ExpPLEPinnedHostEmbedding",
|
||||
ids: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
_lookup_nvfp4_ple_embedding_kernel[(ids.numel(),)](
|
||||
layer._uva_weight,
|
||||
self._uva_weight_scale.view(torch.uint8),
|
||||
layer.weight_scale_2,
|
||||
ids,
|
||||
output,
|
||||
layer.embedding_dim,
|
||||
layer.shard_indices.org_vocab_start_index,
|
||||
layer.shard_indices.org_vocab_end_index,
|
||||
BLOCK_D=triton.next_power_of_2(layer.embedding_dim),
|
||||
)
|
||||
|
||||
def dequantize(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
embeddings: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
del layer
|
||||
# The lookup already decoded the rows to the activation dtype.
|
||||
return embeddings.to(output_dtype)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _lookup_nvfp4_ple_embedding_kernel(
|
||||
weight_ptr,
|
||||
scale_ptr,
|
||||
global_scale_ptr,
|
||||
ids_ptr,
|
||||
output_ptr,
|
||||
embedding_dim,
|
||||
vocab_start,
|
||||
vocab_end,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Decode owned NVFP4 PLE rows; write zeros for rows another rank owns."""
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
idx = tl.load(ids_ptr + row).to(tl.int64)
|
||||
owned = (idx >= vocab_start) & (idx < vocab_end)
|
||||
local_idx = tl.where(owned, idx - vocab_start, 0)
|
||||
offsets = tl.arange(0, BLOCK_D)
|
||||
in_row = offsets < embedding_dim
|
||||
mask = owned & in_row
|
||||
packed = tl.load(
|
||||
weight_ptr + local_idx * (embedding_dim // 2) + offsets // 2,
|
||||
mask=mask,
|
||||
other=0,
|
||||
)
|
||||
# The low nibble holds the even element and the high nibble the odd one.
|
||||
codes = (packed >> ((offsets % 2) * 4)) & 0xF
|
||||
scales = _e4m3_inline(
|
||||
tl.load(
|
||||
scale_ptr + local_idx * (embedding_dim // 16) + offsets // 16,
|
||||
mask=mask,
|
||||
other=0,
|
||||
)
|
||||
)
|
||||
global_scale = tl.load(global_scale_ptr).to(tl.float32)
|
||||
values = _e2m1_inline(codes) * scales * global_scale
|
||||
tl.store(output_ptr + row * embedding_dim + offsets, values, mask=in_row)
|
||||
|
||||
|
||||
class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
|
||||
"""PLE table allocated on the active model device."""
|
||||
|
||||
@@ -358,16 +575,16 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
|
||||
"""Allocate the complete PLE weight on the active device."""
|
||||
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Resident embedding prefetch is a no-op."""
|
||||
return None
|
||||
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Reject prefetch finalization for resident embeddings."""
|
||||
raise RuntimeError("Device PLE embedding does not support prefetch")
|
||||
|
||||
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Gather ETP inputs, look up embeddings, and select local rows."""
|
||||
"""Gather ETP IDs, optionally reduce lookup results, and select local rows."""
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
embeddings = super().forward(gathered_ids)
|
||||
@@ -429,6 +646,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
@@ -442,10 +660,9 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
num_ngram_heads=num_ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
self._row_bytes = self.embedding_dim * self.weight.element_size()
|
||||
self._block_d = triton.next_power_of_2(self._row_bytes)
|
||||
self._prefetch_stream: torch.cuda.Stream | None = None
|
||||
self._prefetch_buffer: torch.Tensor | None = None
|
||||
self._prefetch_alloc_lock = threading.Lock()
|
||||
@@ -473,35 +690,28 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
input_ids: torch.Tensor,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Look up local ETP rows while preserving the weight storage dtype."""
|
||||
"""Look up local ETP rows in the embedding method's lookup dtype."""
|
||||
expected_shape = (*input_ids.shape, self.embedding_dim)
|
||||
lookup_dtype = self.embedding_method.lookup_dtype(self)
|
||||
if output is None:
|
||||
output = torch.empty(
|
||||
expected_shape,
|
||||
dtype=self.weight.dtype,
|
||||
dtype=lookup_dtype,
|
||||
device=input_ids.device,
|
||||
)
|
||||
elif (
|
||||
tuple(output.shape) != expected_shape
|
||||
or output.dtype != self.weight.dtype
|
||||
or output.dtype != lookup_dtype
|
||||
or output.device != input_ids.device
|
||||
):
|
||||
raise ValueError(
|
||||
"PLE prefetch output must match the input shape, weight dtype, "
|
||||
"PLE prefetch output must match the input shape, lookup dtype, "
|
||||
"and input device"
|
||||
)
|
||||
|
||||
flat_ids = input_ids.reshape(-1).long()
|
||||
if flat_ids.numel():
|
||||
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
|
||||
self._uva_weight,
|
||||
flat_ids,
|
||||
output,
|
||||
self._row_bytes,
|
||||
self.shard_indices.org_vocab_start_index,
|
||||
self.shard_indices.org_vocab_end_index,
|
||||
BLOCK_D=self._block_d,
|
||||
)
|
||||
self.embedding_method.lookup_from_pinned(self, flat_ids, output)
|
||||
return output
|
||||
|
||||
def sync_lookup(self, ngram_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -528,11 +738,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
return self.parallel_group.all_reduce(embeddings)
|
||||
|
||||
@eager_break_during_capture
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
def start_prefetch(self, ngram_ids: torch.Tensor) -> None:
|
||||
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
@@ -555,7 +761,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
self._prefetch_rows,
|
||||
self._num_ngram_heads,
|
||||
self.embedding_dim,
|
||||
dtype=self.weight.dtype,
|
||||
dtype=self.embedding_method.lookup_dtype(self),
|
||||
device=self._uva_weight.device,
|
||||
)
|
||||
self._prefetch_buffer = buffer
|
||||
@@ -581,14 +787,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Join the side stream, reduce ETP shards, and select local rows."""
|
||||
"""Join the side stream, optionally reduce ETP shards, and select rows."""
|
||||
prefetch_stream = self._prefetch_stream
|
||||
if prefetch_stream is None:
|
||||
raise RuntimeError("pinned PLE finalize requires a prior start_prefetch")
|
||||
torch.cuda.current_stream().wait_stream(prefetch_stream)
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
|
||||
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
embeddings = self._reduce_etp_embeddings(active_output)
|
||||
embeddings = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
if self.reduce_results:
|
||||
embeddings = self._reduce_etp_embeddings(embeddings)
|
||||
else:
|
||||
# The outer TP reduce-scatter requires ETP=TP.
|
||||
assert self.etp_data_parallel_size == 1
|
||||
embeddings = self._select_embeddings(
|
||||
embeddings,
|
||||
output.shape[0],
|
||||
@@ -596,11 +806,18 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
)
|
||||
output.copy_(embeddings.flatten(-2))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Finish the pinned lookup into graph-owned output storage."""
|
||||
def finalize_prefetch(self, total_tokens: int) -> torch.Tensor:
|
||||
"""Finalize the full-token pinned lookup with optional ETP reduction."""
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
raise RuntimeError("pinned PLE lookup requires a prior start_prefetch")
|
||||
output = buffer.new_empty((hidden_states.shape[0], self._output_dim))
|
||||
# total_tokens counts this DP rank's tokens before SP sharding.
|
||||
output = buffer.new_empty((total_tokens, self._output_dim))
|
||||
self._finalize_prefetch(buffer, output)
|
||||
return output
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
raise RuntimeError(
|
||||
"Pinned-host PLE embedding requires "
|
||||
"start_prefetch() and finalize_prefetch()."
|
||||
)
|
||||
|
||||
@@ -344,6 +344,7 @@ def _ple_conv_kernel(
|
||||
MODE: tl.constexpr,
|
||||
HAS_INIT: tl.constexpr,
|
||||
NULL_STATE_ID: tl.constexpr,
|
||||
FUSE_RESIDUAL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
@@ -438,21 +439,23 @@ def _ple_conv_kernel(
|
||||
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
|
||||
y = conv * tl.sigmoid(conv)
|
||||
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
|
||||
residual = tl.load(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
# Preserve the original eager operation boundaries: short convolution is
|
||||
# first accumulated into the BF16/FP16 PLE output, then the outer residual
|
||||
# is added to that rounded value.
|
||||
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
|
||||
outer_residual = tl.load(
|
||||
outer_residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
|
||||
ple_output = conv_output
|
||||
if FUSE_RESIDUAL:
|
||||
residual = tl.load(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
# Preserve the original eager operation boundaries: short convolution is
|
||||
# first accumulated into the BF16/FP16 PLE output, then the outer residual
|
||||
# is added to that rounded value.
|
||||
ple_output = (residual + conv_output).to(residual_ptr.dtype.element_ty)
|
||||
outer_residual = tl.load(
|
||||
outer_residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
ple_output = outer_residual.to(tl.float32) + ple_output.to(tl.float32)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(
|
||||
@@ -575,7 +578,7 @@ def ple_conv(
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
*,
|
||||
mode: Literal["decode", "spec", "prefill"],
|
||||
dilation: int,
|
||||
@@ -585,7 +588,9 @@ def ple_conv(
|
||||
spec_query_len: int = 1,
|
||||
token_indices: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Add short convolution and the outer residual; update state."""
|
||||
"""Update state and write activated convolution into residual.
|
||||
A non-None outer_residual enables both fused residual additions.
|
||||
"""
|
||||
BLOCK_C = 512
|
||||
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
|
||||
T, C = inputs.shape
|
||||
@@ -657,6 +662,7 @@ def ple_conv(
|
||||
SPEC_QUERY_LEN=kernel_spec_query_len,
|
||||
MODE=mode,
|
||||
HAS_INIT=has_initial_states_arg,
|
||||
FUSE_RESIDUAL=outer_residual is not None,
|
||||
NULL_STATE_ID=NULL_BLOCK_ID,
|
||||
launch_pdl=launch_pdl,
|
||||
num_warps=num_warps,
|
||||
|
||||
@@ -240,12 +240,20 @@ QWEN4_EXP_SM121_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] =
|
||||
8: SkinnyGemmConfig(8, 128, 1, vector_width=2, static_k=6144),
|
||||
16: SkinnyGemmConfig(16, 32, 1, static_k=6144),
|
||||
},
|
||||
# QSA fused QKV/gate projection, TP=2.
|
||||
# QSA fused QKV/gate projection, TP=2 (LoRA keeps the indexer projection separate).
|
||||
(6656, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||
},
|
||||
# QSA fused QKV/gate + replicated indexer Q/K projection, TP=2.
|
||||
(7296, 2560): {
|
||||
1: SkinnyGemmConfig(1, 32, 1, vector_width=2, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 32, 1, vector_width=2, static_k=2560),
|
||||
8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=2560),
|
||||
16: SkinnyGemmConfig(16, 32, 4, vector_width=2, static_k=2560),
|
||||
},
|
||||
# GDN fused QKVZ projection, TP=2.
|
||||
(8192, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
||||
|
||||
@@ -9,8 +9,10 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -37,6 +39,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
MambaStateShapes,
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -68,6 +71,12 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -87,6 +96,14 @@ from .qsa import Qwen4ExpQSAAttention
|
||||
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
|
||||
|
||||
|
||||
def is_hc_sequence_parallel_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Enable GR/PLE SP explicitly or automatically with sequence-parallel MoE."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
return parallel_config.tensor_parallel_size > 1 and (
|
||||
parallel_config.enable_hc_sp or parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
|
||||
|
||||
def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> QuantizationConfig | None:
|
||||
@@ -164,13 +181,19 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
||||
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
||||
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results
|
||||
)
|
||||
if self.replicate_shared_expert and not self.is_sequence_parallel:
|
||||
# Reduce routed outputs before adding replicated shared outputs.
|
||||
# SP then slices the result to avoid summing shared outputs again via RS.
|
||||
self.experts.moe_config.skip_final_all_reduce = False
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.n_shared_experts = int(config.shared_expert_intermediate_size > 0)
|
||||
|
||||
@@ -188,11 +211,12 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
self.config = config
|
||||
self.layer_type = layer_type
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
if vllm_config.parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp HC does not support sequence-parallel MoE"
|
||||
)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.ple: Qwen4ExpPLELayer | None = None
|
||||
ple_layer_ids = config.ple_layer_ids
|
||||
if (self.layer_idx + 1) in ple_layer_ids:
|
||||
@@ -207,6 +231,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_idx=self.layer_idx,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
prefix=f"{prefix}.ple",
|
||||
use_sequence_parallel=self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
if layer_type == "linear_attention":
|
||||
@@ -215,6 +240,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
elif layer_type in _QSA_LAYER_TYPES:
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
@@ -225,6 +251,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
self.self_attn = Qwen4ExpQSAAttention(
|
||||
@@ -233,12 +260,15 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
layer_id=self.layer_idx,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=not self.use_hc_sequence_parallel,
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
@@ -298,6 +328,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states, block_input, injection = attn_hc.mix(hidden_states)
|
||||
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Positions retain the full token axis for both text and MRoPE inputs.
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in _QSA_LAYER_TYPES:
|
||||
@@ -309,10 +342,25 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
raise ValueError("Invalid layer_type")
|
||||
|
||||
mlp_hc = self.mlp_hyper_connection
|
||||
if self.use_hc_sequence_parallel:
|
||||
attn_out = sp_reduce_scatter(attn_out)
|
||||
hidden_states, block_input, injection = mlp_hc.combine_and_mix(
|
||||
hidden_states, attn_out, injection
|
||||
)
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_sequence_parallel:
|
||||
# MoE consumes local tokens and returns complete local outputs.
|
||||
mlp_out = self.mlp(block_input, already_sequence_parallel=True)
|
||||
else:
|
||||
if self.use_hc_sequence_parallel:
|
||||
block_input = sp_all_gather(block_input)[: positions.shape[-1]]
|
||||
mlp_out = self.mlp(block_input)
|
||||
if self.use_hc_sequence_parallel:
|
||||
assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock)
|
||||
reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce
|
||||
# Only sum partial outputs. Reduced outputs already include all ranks.
|
||||
mlp_out = (
|
||||
sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out)
|
||||
)
|
||||
return hidden_states, mlp_out, injection
|
||||
|
||||
|
||||
@@ -376,6 +424,11 @@ class Qwen4ExpModel(nn.Module):
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.use_sequence_parallel = (
|
||||
vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.vocab_size = config.vocab_size
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
@@ -436,7 +489,6 @@ class Qwen4ExpModel(nn.Module):
|
||||
@staticmethod
|
||||
def _start_layer_ple_prefetch(
|
||||
layer: nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
ngram_context: torch.Tensor | None,
|
||||
@@ -447,12 +499,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
return
|
||||
if input_ids is None or query_start_loc is None or ngram_context is None:
|
||||
raise RuntimeError("PLE inputs were not prepared")
|
||||
ple.start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
ple.start_prefetch(input_ids, query_start_loc, ngram_context)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -471,19 +518,31 @@ class Qwen4ExpModel(nn.Module):
|
||||
if input_ids is None:
|
||||
raise ValueError("input_ids or inputs_embeds is required")
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
if self.use_hc_sequence_parallel:
|
||||
if (
|
||||
self.use_sequence_parallel
|
||||
and envs.VLLM_MOE_SKIP_PADDING
|
||||
and is_forward_context_available()
|
||||
):
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
# Expand only the local token rows when SP is enabled.
|
||||
hidden_states = hidden_states.repeat(1, self.config.hc_count)
|
||||
else:
|
||||
if intermediate_tensors is None:
|
||||
raise ValueError("pipeline stage requires intermediate tensors")
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
full_num_tokens = positions.shape[-1]
|
||||
block_output = None
|
||||
injection = None
|
||||
last_layer = None
|
||||
if self.start_layer < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[self.start_layer],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
@@ -495,7 +554,6 @@ class Qwen4ExpModel(nn.Module):
|
||||
if layer_idx + 1 < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[layer_idx + 1],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
@@ -515,6 +573,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
deepstack_embed = deepstack_input_embeds[
|
||||
f"deepstack_input_embeds_{layer_idx}"
|
||||
]
|
||||
if self.use_hc_sequence_parallel:
|
||||
deepstack_embed = sp_shard(deepstack_embed)
|
||||
deepstack_embed = (
|
||||
deepstack_embed.unsqueeze(-2)
|
||||
.expand(
|
||||
@@ -549,6 +609,24 @@ class Qwen4ExpModel(nn.Module):
|
||||
multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Gather LM-head and MTP states together when both are needed.
|
||||
hidden_size = self.config.hidden_size
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, self.config.hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
else:
|
||||
sample_hidden_states = sp_all_gather(sample_hidden_states)[
|
||||
:full_num_tokens
|
||||
]
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
# Capture the pre-final-mixer multi-stream hidden state
|
||||
# [T, hc_count*H] for the MTP drafter (zero extra compute:
|
||||
@@ -604,6 +682,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
class Qwen4ExpForCausalLM(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
@@ -885,6 +964,7 @@ class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
||||
class Qwen4ExpForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration,
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
SupportsReplaySSM,
|
||||
):
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user