[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-39
View File
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V3.2 (``deepseek_v32``) platform entry point.
DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
attend. The same model code serves any DSA checkpoint, including GLM-5.2
(``glm_moe_dsa``), which reuses this architecture.
The CUDA and ROCm implementations select capability-specific kernels
internally and fall back when an optimization is unavailable. Other platforms
use the generic implementation by default.
"""
from vllm.platforms import current_platform
if current_platform.is_cuda():
# GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized
# kernels remain gated on the device capabilities they support.
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
from .nvidia.mtp import DeepseekV32MTP
elif current_platform.is_rocm():
from .amd.model import DeepseekV32ForCausalLM
from .amd.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
from .amd.mtp import DeepseekV32MTP
else:
# XPU and CPU keep the generic implementation.
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
)
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
__all__ = [
"DeepseekV32ForCausalLM",
"DeepseekV32MTP",
"GlmMoeDsaForCausalLM",
]
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-399
View File
@@ -1,399 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
from itertools import islice
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2ForCausalLM,
DeepseekV2MLP,
DeepseekV2MoE,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from vllm.model_executor.models.utils import (
PPMissingLayer,
get_pp_missing_layer_names,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
)
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
from vllm.sequence import IntermediateTensors
from .rocm import DeepseekV32MLAAttention
class DeepseekV32DecoderLayer(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str,
config=None,
topk_indices_buffer: torch.Tensor | None = None,
) -> None:
super().__init__()
if config is None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
self.hidden_size = config.hidden_size
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
layer_idx = int(prefix.split(sep=".")[-1])
self.layer_idx = layer_idx
self.use_mha = False
self.self_attn = DeepseekV32MLAAttention(
vllm_config=vllm_config,
config=config,
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
)
self.zero_mlp_output = False
if (
config.n_routed_experts is not None
and layer_idx >= config.first_k_dense_replace
and layer_idx % moe_layer_freq == 0
):
self.mlp = DeepseekV2MoE(
config=config,
parallel_config=parallel_config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
moe_cfg = self.mlp.experts.moe_config
pc = moe_cfg.moe_parallel_config
# Validated on MoRI only. DeepEP, pplx and the other combines
# keep the previous always-skip behavior.
if pc.use_mori_kernels:
# Combine is already all-reduced; the next fused norm sums TP.
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
else:
moe_cfg.skip_final_all_reduce = True
else:
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.input_layernorm
)
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.post_attention_layernorm
)
hidden_states = self.mlp(hidden_states)
if self.zero_mlp_output:
hidden_states = torch.zeros_like(hidden_states)
return hidden_states, residual
class DeepseekV32Model(torch.nn.Module):
fall_back_to_pt_during_load = False
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
from vllm.platforms import current_platform
self.device = current_platform.device_type
self.vocab_size = config.vocab_size
self.is_v32 = True
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=self.device,
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV32DecoderLayer(
vllm_config=vllm_config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV2MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
self.aux_hidden_state_layers = tuple[int, ...]()
self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
assert input_ids is not None
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = []
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if idx in self.aux_hidden_state_layers:
aux_hidden_states.append(hidden_states + residual)
hidden_states, residual = layer(positions, hidden_states, residual)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = fused_allreduce_rms_norm(hidden_states, residual, self.norm)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
("fused_qkv_a_proj", "q_a_proj", 0),
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
("wk_weights_proj", "wk", 0),
("wk_weights_proj", "weights_proj", 1),
]
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_pending_wk_fp8: dict = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
continue
is_fusion_moe_shared_experts_layer = (
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
_pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
) and name_mapped not in params_dict:
continue
name = name_mapped
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
param.weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
# When AITER fusion_shared_experts is enabled the checkpoint
# provides a single widened shared_experts tensor per layer with
# no expert index. Split it evenly and route the slices into the
# expert slots appended after the routed experts.
num_chunks = 1
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
# gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1).
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
total = loaded_weight.shape[split_dim]
assert total % num_chunks == 0, (
f"Shared expert weight dim {total} "
f"not divisible by num_chunks {num_chunks}"
)
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in chunk_name:
continue
is_expert_weight = True
name_mapped = chunk_name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
loader = getattr(param, "weight_loader", default_weight_loader)
loader(param, loaded_weight)
if name is not None and not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
return loaded_params
class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
model_cls = DeepseekV32Model
def set_moe_parameters(self):
self.num_expert_groups = getattr(self.config, "n_group", 1)
self.moe_layers = []
self.moe_mlp_layers = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
if isinstance(layer.mlp, DeepseekV2MoE):
example_moe = layer.mlp
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
-425
View File
@@ -1,425 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2MixtureOfExperts,
DeepseekV2MoE,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
get_pp_missing_layer_names,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .model import DeepseekV32DecoderLayer
class DeepseekV32MultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=current_platform.device_type,
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
self.mtp_block = DeepseekV32DecoderLayer(
vllm_config,
prefix,
config=config,
topk_indices_buffer=topk_indices_buffer,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
eh_input = fused_eh_norm(
positions,
inputs_embeds,
previous_hidden_states,
self.enorm.weight,
self.hnorm.weight,
self.enorm.variance_epsilon,
)
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = residual + hidden_states
hidden_states = self.shared_head.norm(hidden_states)
return hidden_states, hidden_states
class DeepseekV32MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
self.layers = torch.nn.ModuleDict(
{
str(idx): DeepseekV32MultiTokenPredictorLayer(
vllm_config, f"{prefix}.layers.{idx}"
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
def set_skip_topk(self, skip: bool):
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "skip_topk"):
self_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor):
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
num_slots = slot_ids.numel()
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
self.model = DeepseekV32MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.set_moe_parameters()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], self.config.hidden_size
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def set_moe_parameters(self):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers = []
self.moe_mlp_layers = []
example_moe = None
for layer in self.model.layers.values():
mlp = layer.mtp_block.mlp
if isinstance(mlp, DeepseekV2MoE):
example_moe = mlp
self.moe_mlp_layers.append(mlp)
self.moe_layers.append(mlp.experts)
self.extract_moe_parameters(example_moe)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
return self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"eh_proj",
"shared_head",
]
shared_weight_names = ["embed_tokens"]
spec_layer_weight = False
shared_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
if weight_name in shared_weight_names:
shared_weight = True
break
if not spec_layer_weight:
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
elif shared_weight:
name = name.replace(f"model.layers.{spec_layer}.", "model.")
return name
def is_unused_checkpoint_weight(self, name: str) -> bool:
return (
get_spec_layer_idx_from_weight_name(self.config, name) is None
and "embed_tokens" not in name
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
("fused_qkv_a_proj", "q_a_proj", 0),
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
("wk_weights_proj", "wk", 0),
("wk_weights_proj", "weights_proj", 1),
]
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if self.is_fused_shared_expert_enabled
else 0
),
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_pending_wk_fp8: dict = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
continue
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
# A tied top-level embed_tokens has no spec layer to rewrite
# from; the draft needs its own copy under PP.
param = params_dict.get(name) if "embed_tokens" in name else None
if param is not None:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
is_fusion_moe_shared_experts_layer = (
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
_pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
) and name_mapped not in params_dict:
continue
else:
name = name_mapped
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
num_chunks = 1
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
total = loaded_weight.shape[split_dim]
assert total % num_chunks == 0
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in chunk_name:
continue
is_expert_weight = True
name_mapped = chunk_name.replace(weight_name, param_name)
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if (
spec_layer != self.model.mtp_start_layer_idx
and ".layers" not in name
):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
if not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
loaded_layers: set[int] = set()
for param_name in loaded_params:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
if spec_layer is not None:
loaded_layers.add(spec_layer)
for layer_idx in range(
self.model.mtp_start_layer_idx,
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
):
if layer_idx not in loaded_layers:
raise ValueError(
f"MTP speculative decoding layer {layer_idx} weights "
f"missing from checkpoint."
)
return loaded_params
-314
View File
@@ -1,314 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
def get_attn_backend(self):
return DeepseekV32IndexerBackend
@property
def uses_shuffled_layout(self) -> bool:
# aiter's gather/insert pair shuffles the cache above block size 1:
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1
class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
indexer_cache_cls = DeepseekV32ROCmIndexerCache
class DeepseekV32MLAAttention(DeepseekV32Attention):
indexer_cls = DeepseekV32ROCmIndexer
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
super().__init__(
vllm_config,
config,
prefix,
topk_indices_buffer,
attn_backend=ROCMAiterMLASparseBackend,
)
self.indexer_op: SparseAttnIndexer | None = None
if self.indexer is not None:
self.indexer_op = SparseAttnIndexer(
self.indexer.k_cache,
self.indexer.quant_block_size,
self.indexer.scale_fmt,
self.indexer.topk_tokens,
self.indexer.head_dim,
self.indexer.max_model_len,
self.indexer.max_total_seq_len,
topk_indices_buffer,
skip_k_cache_insert=True,
)
self._fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype)
self._fp8_kv_needs_view = self._fp8_kv and self.kv_cache_dtype != "fp8_ds_mla"
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_c, k_pe = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
if self.indexer is not None and not self.skip_topk:
kw = self.indexer.wk_weights_proj(hidden_states)[0]
index_k = kw[:, : self.indexer.head_dim]
index_weights = kw[:, self.indexer.head_dim :]
else:
index_k = None
index_weights = None
num_tokens = hidden_states.shape[0]
output = torch.empty(
(num_tokens, self.num_local_heads * self.v_head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._fused_attention(
positions, q_c, kv_c, k_pe, index_k, index_weights, output
)
return self.o_proj(output)[0]
def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
q_nope = q_nope.transpose(0, 1) # (N, tokens, P)
if self.is_aiter_triton_fp4_bmm_enabled:
ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
)
elif self.is_aiter_triton_fp8_bmm_enabled:
ql_nope = rocm_aiter_ops.triton_fp8_bmm(
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
)
else:
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
return ql_nope, q_pe
def _run_indexer(
self,
q_c: torch.Tensor,
index_q_fp8: torch.Tensor | None,
index_weights_out: torch.Tensor | None,
) -> None:
"""Run the ROCm sparse indexer (forward_hip) if this layer has an indexer."""
if self.indexer_op is not None:
self.indexer_op.forward_hip(q_c, index_q_fp8, None, index_weights_out)
def _build_q_for_attn(
self,
ql_nope: torch.Tensor,
mqa_q: torch.Tensor,
num_actual: int,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if self._fp8_kv:
# fp8 KV: mqa_q is the full [ql_nope; q_pe] packed as fp8.
return mqa_q[:num_actual]
return (ql_nope[:num_actual], mqa_q[:num_actual])
def _compute_uv_out(
self,
attn_out: torch.Tensor,
output: torch.Tensor,
num_actual: int,
) -> None:
x = attn_out.view(
num_actual, self.num_local_heads, self.kv_lora_rank
).transpose(0, 1) # (N, tokens, L)
out_view = output[:num_actual].view(
num_actual, self.num_local_heads, self.v_head_dim
)
if self.is_aiter_triton_fp4_bmm_enabled:
rocm_aiter_ops.batched_gemm_a16wfp4(
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
)
elif self.is_aiter_triton_fp8_bmm_enabled:
rocm_aiter_ops.triton_fp8_bmm(
x,
self.W_V,
self.W_V_scale,
group_size=128,
transpose_bm=True,
YQ=out_view,
)
else:
torch.bmm(x, self.W_UV, out=out_view.transpose(0, 1))
@eager_break_during_capture
def _fused_attention(
self,
positions: torch.Tensor,
q_c: torch.Tensor,
kv_c: torch.Tensor,
k_pe: torch.Tensor,
index_k: torch.Tensor | None,
index_weights: torch.Tensor | None,
output: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.layer_name)
elif isinstance(attn_metadata_raw, list):
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
else:
attn_metadata = attn_metadata_raw
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
if self.indexer is not None and not self.skip_topk:
has_indexer = True
indexer_k_norm_w = self.indexer.k_norm.weight
indexer_k_norm_bias = self.indexer.k_norm.bias
indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_k_cache = self.indexer.k_cache.kv_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5
else:
has_indexer = False
indexer_k_norm_w = None
indexer_k_norm_bias = None
indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None
indexer_cache_shuffled = False
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0
if attn_metadata is None:
mla_kv_cache = None
mla_k_scale = None
indexer_k_cache = None
mla_slot = None
else:
mla_kv_cache = self.kv_cache
mla_k_scale = self._k_scale
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
kv_c_out = torch.empty_like(kv_c) if use_mha else None
k_pe_out = torch.empty_like(k_pe) if use_mha else None
q_c = fused_norm_rope(
positions,
q_c,
self.q_a_layernorm.weight,
self.q_a_layernorm.variance_epsilon,
kv_c,
self.kv_a_layernorm.weight,
self.kv_a_layernorm.variance_epsilon,
k_pe,
self.rotary_emb.cos_sin_cache,
index_k,
indexer_k_norm_w,
indexer_k_norm_bias,
indexer_k_norm_eps,
indexer_k_rope_cos_sin_cache,
self.topk_indices_buffer,
slot_mapping=mla_slot,
indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
kv_c_out=kv_c_out,
k_pe_out=k_pe_out,
)
if use_mha:
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split(
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
)
ql_nope = q_nope
else:
ql_nope, q_pe = self._compute_ql_nope(q_c)
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
else:
index_q = None
index_q_fp8, index_weights_out, mqa_q = fused_q(
positions,
q_pe,
self.rotary_emb.cos_sin_cache,
index_q,
self.indexer_rope_emb.cos_sin_cache if has_indexer else None,
ql_nope,
self._q_scale,
index_weights,
indexer_softmax_scale,
indexer_n_head_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
quantize_mqa=self._fp8_kv and not use_mha,
)
if self.indexer is not None and not self.skip_topk:
self._run_indexer(q_c, index_q_fp8, index_weights_out)
if attn_metadata is None:
output.zero_()
return
if use_mha:
assert kv_c_out is not None and k_pe_out is not None
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
self.forward_impl(
mha_q,
kv_c_out,
k_pe_out.unsqueeze(1),
self.kv_cache,
attn_metadata,
output,
)
return
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
kv_cache = self.kv_cache
if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(current_platform.fp8_dtype())
q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual)
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
q_for_attn, kv_cache, attn_metadata, self
)
self._compute_uv_out(attn_out, output, num_actual)
-653
View File
@@ -1,653 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, ClassVar, cast
import torch
import torch.nn as nn
from transformers import DeepseekV2Config, DeepseekV3Config
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import (
SparseAttnIndexer,
sparse_attn_indexer,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepSeekV2FusedQkvAProjLinear,
DeepseekV32IndexerCache,
yarn_get_mscale,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.ops.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
)
class DeepseekV32Indexer(nn.Module):
indexer_cache_cls = DeepseekV32IndexerCache
def __init__(
self,
vllm_config: VllmConfig,
config: DeepseekV2Config | DeepseekV3Config,
hidden_size: int,
q_lora_rank: int,
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
topk_indices_buffer: torch.Tensor | None,
prefix: str = "",
):
super().__init__()
self.topk_tokens = config.index_topk
self.n_head = config.index_n_heads
self.head_dim = config.index_head_dim
self.rope_dim = config.qk_rope_head_dim
self.q_lora_rank = q_lora_rank
self.wq_b = ReplicatedLinear(
self.q_lora_rank,
self.head_dim * self.n_head,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.wq_b",
)
self.wk_weights_proj = MergedColumnParallelLinear(
hidden_size,
[self.head_dim, self.n_head],
bias=False,
quant_config=None,
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
self.softmax_scale = self.head_dim**-0.5
self.scale_fmt = "ue8m0"
self.quant_block_size = 128
self.topk_indices_buffer = topk_indices_buffer
assert cache_config is not None, "DeepSeek V3.2 indexer requires cache_config"
self.k_cache = type(self).indexer_cache_cls(
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
dtype=torch.uint8,
prefix=f"{prefix}.k_cache",
cache_config=cache_config,
)
self.max_model_len = vllm_config.model_config.max_model_len
self.prefix = prefix
from vllm.v1.attention.backends.mla.indexer import (
get_max_prefill_buffer_size,
)
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
self.indexer_op = SparseAttnIndexer(
self.k_cache,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
)
class DeepseekV32Attention(MLAAttention):
indexer: "DeepseekV32Indexer | None"
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
supports_pcp_dcp: ClassVar[bool] = True
def __init__(
self,
vllm_config: VllmConfig,
config: DeepseekV2Config | DeepseekV3Config,
prefix: str,
topk_indices_buffer: torch.Tensor | None = None,
attn_backend: "type | None" = None,
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
) -> None:
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
hidden_size = config.hidden_size
qk_nope_head_dim = config.qk_nope_head_dim
qk_rope_head_dim = config.qk_rope_head_dim
v_head_dim = config.v_head_dim
q_lora_rank = config.q_lora_rank
kv_lora_rank = config.kv_lora_rank
num_heads = config.num_attention_heads
tp_size = get_tensor_model_parallel_world_size()
assert num_heads % tp_size == 0
num_local_heads = num_heads // tp_size
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
scaling = qk_head_dim**-0.5
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
# DSA checkpoints may use plain ("default") or yarn-scaled RoPE.
if config.rope_parameters["rope_type"] != "default":
config.rope_parameters["rope_type"] = (
"deepseek_llama_scaling"
if config.rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
if config.rope_parameters["rope_type"] == "deepseek_yarn":
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
scaling_factor = config.rope_parameters["factor"]
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
scaling = scaling * mscale * mscale
layer_id = extract_layer_index(prefix)
index_topk_freq = getattr(config, "index_topk_freq", 1)
index_topk_pattern = getattr(config, "index_topk_pattern", None)
index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
if index_topk_pattern is None:
skip_topk = (
max(layer_id - index_skip_topk_offset + 1, 0) % index_topk_freq != 0
)
elif 0 <= layer_id < len(index_topk_pattern):
skip_topk = index_topk_pattern[layer_id] == "S"
else:
skip_topk = False
num_hidden_layers = getattr(config, "num_hidden_layers", None)
is_mtp_layer = num_hidden_layers is not None and layer_id >= num_hidden_layers
kv_b_proj = ColumnParallelLinear(
kv_lora_rank,
num_heads * (qk_nope_head_dim + v_head_dim),
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_b_proj",
)
indexer = None
if not skip_topk or is_mtp_layer:
indexer = type(self).indexer_cls(
vllm_config,
config,
hidden_size,
q_lora_rank,
quant_config,
cache_config,
topk_indices_buffer,
prefix=f"{prefix}.indexer",
)
super().__init__(
num_heads=num_local_heads,
scale=scaling,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=q_lora_rank,
kv_lora_rank=kv_lora_rank,
kv_b_proj=kv_b_proj,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
use_sparse=True,
indexer=indexer,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
attn_backend=attn_backend,
)
self.num_local_heads = num_local_heads
self.qk_head_dim = qk_head_dim
self.indexer = indexer
self.topk_indices_buffer = topk_indices_buffer
self.skip_topk = False
enable_short_prefill_scoring_skip = (
not is_mtp_layer
and not skip_topk
and not self.use_pcp
and current_platform.is_cuda()
and self.supports_dense_mha_prefill
)
self._dense_mha_metadata_layer_name = (
self.layer_name if enable_short_prefill_scoring_skip else ""
)
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
self._fp8_kv_needs_view = fp8_attention and not self.kv_cache_dtype.endswith(
"_ds_mla"
)
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
# Remaining MLA projections (registered on this module).
self.fused_qkv_a_proj = DeepSeekV2FusedQkvAProjLinear(
hidden_size,
[q_lora_rank, kv_lora_rank + qk_rope_head_dim],
quant_config=quant_config,
prefix=f"{prefix}.fused_qkv_a_proj",
)
self.q_a_layernorm = RMSNorm(q_lora_rank, eps=config.rms_norm_eps)
self.q_b_proj = ColumnParallelLinear(
q_lora_rank,
num_heads * qk_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.q_b_proj",
)
self.kv_a_layernorm = RMSNorm(kv_lora_rank, eps=config.rms_norm_eps)
self.o_proj = RowParallelLinear(
num_heads * v_head_dim,
hidden_size,
bias=False,
reduce_results=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
is_neox_style=False,
)
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
)
def _should_prepare_mqa_query(
self,
attn_metadata: "MLACommonMetadata | None",
cudagraph_runtime_mode: CUDAGraphMode,
) -> bool:
return (
attn_metadata is None
or not self._use_sparse_mha(attn_metadata)
or cudagraph_runtime_mode == CUDAGraphMode.FULL
or torch.cuda.is_current_stream_capturing()
)
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_c, k_pe = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
if self.indexer is not None and not self.skip_topk:
kw = self.indexer.wk_weights_proj(hidden_states)[0]
index_k = kw[:, : self.indexer.head_dim]
index_weights = kw[:, self.indexer.head_dim :]
else:
index_k = None
index_weights = None
num_tokens = hidden_states.shape[0]
output = torch.empty(
(num_tokens, self.num_local_heads * self.v_head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
forward_context = get_forward_context()
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
indexer_slot = (
slot_mapping.get(self.indexer.k_cache.prefix)
if self.indexer is not None
else None
)
hisparse_cache = self.hisparse_cache
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
self.impl.prepare_for_batch(layer_attn_metadata)
prepare_mqa_query = self._should_prepare_mqa_query(
layer_attn_metadata, forward_context.cudagraph_runtime_mode
)
if self.indexer is not None and not self.skip_topk:
has_indexer = True
indexer_k_norm_w = self.indexer.k_norm.weight
indexer_k_norm_bias = self.indexer.k_norm.bias
indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
index_k_out = torch.empty_like(index_k) if self.use_pcp else None
indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5
else:
has_indexer = False
indexer_k_norm_w = None
indexer_k_norm_bias = None
indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None
indexer_cache_shuffled = False
index_k_out = None
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0
if forward_context.attn_metadata is None or self.use_pcp:
mla_kv_cache = None
mla_k_scale = None
indexer_k_cache = None
mla_slot = None
indexer_slot = None
else:
mla_kv_cache = None if hisparse_cache is not None else self.kv_cache
mla_k_scale = self._k_scale
if self.use_pcp or hisparse_cache is None:
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
else:
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
q_c = fused_norm_rope(
positions,
q_c,
self.q_a_layernorm.weight,
self.q_a_layernorm.variance_epsilon,
kv_c,
self.kv_a_layernorm.weight,
self.kv_a_layernorm.variance_epsilon,
k_pe,
self.rotary_emb.cos_sin_cache,
index_k,
indexer_k_norm_w,
indexer_k_norm_bias,
indexer_k_norm_eps,
indexer_k_rope_cos_sin_cache,
self.topk_indices_buffer,
slot_mapping=mla_slot,
indexer_slot_mapping=indexer_slot,
indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
kv_c_out=kv_c_out,
k_pe_out=k_pe_out,
index_k_out=index_k_out,
)
if hisparse_cache is not None and mla_slot is not None:
assert kv_c_out is not None and k_pe_out is not None
self.update_kv_cache(
kv_c_out,
k_pe_out,
self.kv_cache,
mla_slot,
layer_attn_metadata,
self.kv_cache_dtype,
self._k_scale,
)
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
if prepare_mqa_query:
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
else:
ql_nope = q_nope
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
else:
index_q = None
index_q_fp8, index_weights_out, mqa_q = fused_q(
positions,
q_pe,
self.rotary_emb.cos_sin_cache,
index_q,
self.indexer_rope_emb.cos_sin_cache if has_indexer else None,
ql_nope,
self._q_scale,
index_weights,
indexer_softmax_scale,
indexer_n_head_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
quantize_mqa=self._fp8_query and prepare_mqa_query,
)
self._sparse_indexer_and_attn(
positions,
q_c,
q_nope,
q_pe,
index_q_fp8,
index_k_out,
index_weights_out,
kv_c_out,
k_pe_out,
ql_nope,
mqa_q,
output,
)
return self.o_proj(output)[0]
@eager_break_during_capture
def _sparse_indexer_and_attn(
self,
positions: torch.Tensor,
q_c: torch.Tensor,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
index_q_fp8: torch.Tensor | None,
index_k: torch.Tensor | None,
index_weights_out: torch.Tensor | None,
kv_c: torch.Tensor | None,
k_pe: torch.Tensor | None,
ql_nope: torch.Tensor,
mqa_q: torch.Tensor,
output: torch.Tensor,
) -> None:
if self.indexer is not None and not self.skip_topk:
assert index_q_fp8 is not None
assert index_weights_out is not None
if self.use_pcp:
assert index_k is not None
sparse_attn_indexer(
q_c,
self.indexer.k_cache.prefix,
self.indexer.k_cache.kv_cache,
index_q_fp8,
None,
index_k,
index_weights_out,
self.indexer.quant_block_size,
self.indexer.scale_fmt,
self.indexer.topk_tokens,
self.indexer.head_dim,
self.indexer.max_model_len,
self.indexer.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=not self.use_pcp,
use_pcp=self.use_pcp,
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name,
dcp_rank=(
self.dcp_manager.group.rank_in_group
if self.dcp_manager is not None
else 0
),
dcp_world_size=(
self._vllm_config.parallel_config.decode_context_parallel_size
),
cp_kv_cache_interleave_size=(
self._vllm_config.parallel_config.cp_kv_cache_interleave_size
),
skip_topk_buffer_clear=True,
topk_backend=self.indexer.indexer_op.topk_backend,
)
self.impl.record_logical_topk_ready() # type: ignore[attr-defined]
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
self.layer_name
)
if attn_metadata is None:
output.zero_()
return
attn_metadata = cast("MLACommonMetadata", attn_metadata)
self.impl.prepare_for_batch(attn_metadata)
if self.use_pcp:
assert kv_c is not None and k_pe is not None
kv_for_cache, kpe_for_cache, cache_slot_mapping = (
maybe_gather_mla_latent_cache_inputs(
kv_c,
k_pe.unsqueeze(1),
layer_slot_mapping,
attn_metadata.num_decode_tokens,
True,
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
)
)
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_for_cache,
kpe_for_cache,
kv_cache,
cache_slot_mapping,
self.kv_cache_dtype,
self._k_scale,
)
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
if num_actual == 0:
output.zero_()
return
if self._use_sparse_mha(attn_metadata):
assert kv_c is not None and k_pe is not None
mha_q_pe = (
self.rotary_emb(positions, q_pe)[0]
if mqa_q.dtype != q_pe.dtype
else mqa_q
)
mha_q = torch.cat((q_nope, mha_q_pe), dim=-1)
self.forward_impl(
mha_q,
kv_c,
k_pe.unsqueeze(1),
kv_cache,
attn_metadata,
output,
)
return
if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(current_platform.fp8_dtype())
if self._fp8_query:
# FlashInfer sparse: single packed fp8 query.
mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[
:num_actual
]
else:
mqa_q_arg = (ql_nope[:num_actual], mqa_q[:num_actual])
if self.use_pcp and self.impl.dcp_world_size > self.impl.pcp_world_size:
if isinstance(mqa_q_arg, tuple):
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
elif not self.use_pcp and self.impl.dcp_world_size > 1:
assert self.dcp_manager is not None
if isinstance(mqa_q_arg, tuple):
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
assert self.dcp_manager.query_gather is not None
mqa_q_arg = self.dcp_manager.query_gather(mqa_q_arg)
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
mqa_q_arg, kv_cache, attn_metadata, self
)
if self.impl.dcp_world_size > 1:
assert lse is not None and self.dcp_manager is not None
seq_lens: torch.Tensor | None
query_start_loc: torch.Tensor | None
if self.use_pcp:
decode_metadata = getattr(attn_metadata, "decode", None)
if decode_metadata is not None:
seq_lens = decode_metadata.seq_lens
else:
all_seq_lens = cast(
torch.Tensor,
attn_metadata.seq_lens, # type: ignore[attr-defined]
)
seq_lens = all_seq_lens[: attn_metadata.num_decodes]
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
else:
# The backend emits (0, -inf) for empty local shards, so no
# PCP-only empty-shard metadata is needed.
seq_lens = None
query_start_loc = None
# Under PCP+DCP the prefill rows attended over the gathered KV, so
# only the decode rows carry an LSE and take part in the merge.
num_merge_rows = lse.shape[0]
if num_merge_rows == attn_out.shape[0]:
attn_out = self.dcp_manager.combine(
attn_out,
lse,
seq_lens=seq_lens, # type: ignore[arg-type]
query_start_loc=query_start_loc, # type: ignore[arg-type]
)
elif num_merge_rows > 0:
attn_out[:num_merge_rows] = self.dcp_manager.combine(
attn_out[:num_merge_rows],
lse,
seq_lens=seq_lens, # type: ignore[arg-type]
query_start_loc=query_start_loc, # type: ignore[arg-type]
)
if self.use_pcp:
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
# NOTE(woosuk): While the below does not need to be in the eager region,
# we put it here to avoid copying the attention output. Move this back to the
# captured region once forward_mqa supports `out` argument.
x = attn_out.view(
num_actual, self.num_local_heads, self.kv_lora_rank
).transpose(0, 1)
out = (
output[:num_actual]
.view(num_actual, self.num_local_heads, self.v_head_dim)
.transpose(0, 1)
)
torch.bmm(x, self.W_UV, out=out)
if self.use_pcp and num_actual < output.shape[0]:
output[num_actual:].zero_()
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,199 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
import torch
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.platforms import current_platform
Backend = Literal["cute", "dsv3_fused_a"]
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
@dataclass(frozen=True, slots=True)
class GLM52ProjectionSpec:
n: int
k: int
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...]
dsv3_tokens: frozenset[int] = frozenset()
def build_plan(self) -> dict[int, ResolvedCall]:
plan: dict[int, ResolvedCall] = {
num_tokens: ("cute", config) for num_tokens, config in self.cute_configs
}
plan.update(
(num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens
)
return plan
GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec(
n=2624,
k=6144,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=6144)),
(2, SkinnyGemmConfig(2, 128, 2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
GLM52_Q_B_PROJECTION = GLM52ProjectionSpec(
n=2048,
k=2048,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=2048)),
(2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
# The MTP eh_proj is a plain nn.Linear, so it gets its plan through
# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4.
GLM52_EH_PROJECTION = GLM52ProjectionSpec(
n=6144,
k=12288,
cute_configs=(
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)),
(2, SkinnyGemmConfig(2, 64, 2)),
(3, SkinnyGemmConfig(3, 64, 2)),
),
)
GLM52_PROJECTIONS = {
(spec.n, spec.k): spec
for spec in (
GLM52_QKV_A_PROJECTION,
GLM52_Q_B_PROJECTION,
GLM52_EH_PROJECTION,
)
}
def _is_sm103() -> bool:
return current_platform.is_device_capability((10, 3))
def _is_supported_row_major(tensor: torch.Tensor) -> bool:
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
not envs.VLLM_BATCH_INVARIANT
and _is_supported_row_major(x)
and _is_supported_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and x.is_cuda
and weight.is_cuda
and x.device == weight.device
and x.shape[1] == weight.shape[1]
)
def run_glm52_plan(
plan: dict[int, ResolvedCall] | None,
x: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor | None:
if plan is None or not _runtime_ok(x, weight):
return None
entry = plan.get(x.shape[0])
if entry is None:
return None
backend, config = entry
if backend == "cute":
if not shape_dynamic_skinny_gemm.is_available():
return None
return shape_dynamic_skinny_gemm(x, weight, config)
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
return None
output = torch.empty(
(x.shape[0], weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
return output
def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None:
if configs and shape_dynamic_skinny_gemm.is_available():
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs)
def build_glm52_plan(
weight: torch.Tensor | None, dtype: torch.dtype
) -> dict[int, ResolvedCall] | None:
"""Plan for a weight the walk below cannot reach (a plain ``nn.Linear``)."""
if dtype != torch.bfloat16 or not _is_sm103():
return None
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
return None
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
return None
_request_warmup(dtype, {config for _, config in spec.cute_configs})
return spec.build_plan()
class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod):
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
super().__init__()
self._plan = plan
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if bias is None:
output = run_glm52_plan(self._plan, x, layer.weight)
if output is not None:
return output
return super().apply(layer, x, bias)
def enable_glm52_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
if dtype != torch.bfloat16 or not _is_sm103():
return
warmup_configs: set[SkinnyGemmConfig] = set()
for child in module.modules():
if (
not isinstance(child, LinearBase)
or type(child.quant_method) is not UnquantizedLinearMethod
):
continue
weight = getattr(child, "weight", None)
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
continue
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
continue
child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan())
warmup_configs.update(config for _, config in spec.cute_configs)
_request_warmup(dtype, warmup_configs)
-454
View File
@@ -1,454 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
from itertools import islice
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_embed_norm import (
fused_embed_norm,
has_full_vocab_on_rank,
make_input_embedding,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2ForCausalLM,
DeepseekV2MLP,
DeepseekV2MoE,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from vllm.model_executor.models.utils import (
PPMissingLayer,
get_pp_missing_layer_names,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
)
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
class DeepseekV32DecoderLayer(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str,
config=None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
) -> None:
super().__init__()
if config is None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
self.hidden_size = config.hidden_size
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
layer_idx = int(prefix.split(sep=".")[-1])
self.layer_idx = layer_idx
self.use_mha = False
self.use_sequence_parallel = (
parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size == 1
)
self.self_attn = DeepseekV32Attention(
vllm_config=vllm_config,
config=config,
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
if (
config.n_routed_experts is not None
and layer_idx >= config.first_k_dense_replace
and layer_idx % moe_layer_freq == 0
):
# Keep the MoE output un-reduced. Non-SP fuses its all-reduce into
# the next layer norm; SP keeps the token shard local.
self.mlp = DeepseekV2MoE(
config=config,
parallel_config=parallel_config,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.mlp",
apply_routed_scale_to_output=False,
)
else:
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
reduce_results=False,
is_sequence_parallel=self.use_sequence_parallel,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
attn_in: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
full_num_tokens = positions.shape[0]
if residual is None:
# First layer: hidden_states is the embedding (the residual).
residual = hidden_states
# ``attn_in`` is input_layernorm(embedding) already computed fused
# with the embedding gather; otherwise apply it here.
hidden_states = (
attn_in if attn_in is not None else self.input_layernorm(hidden_states)
)
elif self.use_sequence_parallel:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
else:
# The previous layer's MLP/MoE output is left un-reduced; fuse its
# all-reduce into this input_layernorm.
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.input_layernorm
)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
# self_attn's o_proj runs reduce_results=False; reduce before RMSNorm.
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
if self.use_sequence_parallel:
hidden_states = sp_reduce_scatter(hidden_states)
hidden_states, residual = self.post_attention_layernorm(
hidden_states, residual
)
else:
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.post_attention_layernorm
)
if self.use_sequence_parallel and isinstance(self.mlp, DeepseekV2MoE):
hidden_states = self.mlp(hidden_states, already_sequence_parallel=True)
else:
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
class DeepseekV32Model(torch.nn.Module):
fall_back_to_pt_during_load = False
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
from vllm.platforms import current_platform
self.device = current_platform.device_type
self.vocab_size = config.vocab_size
parallel_config = vllm_config.parallel_config
self.use_sequence_parallel = (
parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size == 1
)
# DSA is always sparse (has index_topk); allocate the shared top-k
# buffer the indexer writes and the sparse MLA backend reads.
self.is_v32 = True
# On the model, not a local: the MTP proposer shares it with the draft.
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=self.device,
)
index_group_builder = SparseMLAIndexGroupBuilder(
self.topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if get_pp_group().is_first_rank:
self.embed_tokens = make_input_embedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
)
else:
self.embed_tokens = PPMissingLayer()
# The fused embed+norm gather needs the full table on-rank.
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV32DecoderLayer(
vllm_config=vllm_config,
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
index_group_builder=index_group_builder,
),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
self.aux_hidden_state_layers = tuple[int, ...]()
self.num_redundant_experts = parallel_config.eplb_config.num_redundant_experts
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
attn_in = None
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
elif self.replicated_embed:
assert input_ids is not None
# Full table on-rank: gather the embedding and the first layer's
# input_layernorm in one launch. ``attn_in`` is the pre-normed
# attention input; ``hidden_states`` is the (residual) embedding.
hidden_states, attn_in = fused_embed_norm(
input_ids,
self.embed_tokens.weight,
chain_weight=self.layers[self.start_layer].input_layernorm.weight,
eps=self.config.rms_norm_eps,
)
else:
assert input_ids is not None
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
if attn_in is not None:
attn_in = sp_shard(attn_in)
assert residual is None, "Currently, SP is not supported with PP"
aux_hidden_states = []
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if idx in self.aux_hidden_state_layers:
aux_hidden_states.append(
hidden_states if residual is None else hidden_states + residual
)
hidden_states, residual = layer(positions, hidden_states, residual, attn_in)
attn_in = None
if not get_pp_group().is_last_rank:
assert not self.use_sequence_parallel, (
"Currently, SP is not supported with PP"
)
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if self.use_sequence_parallel:
hidden_states, _ = self.norm(hidden_states, residual)
if aux_hidden_states:
hidden_size = hidden_states.shape[-1]
packed_hidden_states = torch.cat(
[hidden_states, *aux_hidden_states], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)
packed_hidden_states = packed_hidden_states[:full_num_tokens]
hidden_states, *aux_hidden_states = packed_hidden_states.split(
hidden_size, dim=-1
)
else:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
else:
hidden_states, _ = fused_allreduce_rms_norm(
hidden_states, residual, self.norm
)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# DSA-only: MLA (fused_qkv_a_proj) + the fused indexer wk/weights_proj +
# routed experts. No MHA (qkv_proj) or ROCm shared-expert-fusion paths.
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
("fused_qkv_a_proj", "q_a_proj", 0),
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
("wk_weights_proj", "wk", 0),
("wk_weights_proj", "weights_proj", 1),
]
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_redundant_experts=self.num_redundant_experts,
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_pending_wk_fp8: dict = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
# MTP / nextn layers are loaded by the MTP model, not here.
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
continue
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
_pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
# Experts are handled below; skip here before the name rewrite.
if ("mlp.experts." in name) and name not in params_dict:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
) and name_mapped not in params_dict:
continue
name = name_mapped
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
param.weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in name:
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
loader = getattr(param, "weight_loader", default_weight_loader)
loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
"""DSA causal LM — DeepSeek V2/V3 orchestration with the DSA backbone.
Serves DeepSeek V3.2 and any architecture reusing DSA (e.g. GLM-5.2).
"""
model_cls = DeepseekV32Model
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
def set_moe_parameters(self):
# Same as the base, but keyed on the MoE block type rather than the
# decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module).
self.num_expert_groups = getattr(self.config, "n_group", 1)
self.moe_layers = []
self.moe_mlp_layers = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
if isinstance(layer.mlp, DeepseekV2MoE):
example_moe = layer.mlp
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
-494
View File
@@ -1,494 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_embed_norm import (
fused_embed_eh_norm,
has_full_vocab_on_rank,
make_input_embedding,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2MixtureOfExperts,
DeepseekV2MoE,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
get_pp_missing_layer_names,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .glm52_low_latency_gemm import (
build_glm52_plan,
enable_glm52_low_latency_gemm,
run_glm52_plan,
)
from .model import DeepseekV32DecoderLayer
class DeepseekV32MultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self._eh_plan = (
build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype)
if config.model_type == "glm_moe_dsa"
else None
)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=current_platform.device_type,
)
index_group_builder = SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
self.mtp_block = DeepseekV32DecoderLayer(
vllm_config,
prefix,
config=config,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
embed_table: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
# Fused zero pos-0 + enorm(embeds) + hnorm(prev) + cat -> [N, 2H]. With a
# replicated table the caller passes ``embed_table`` so the embedding
# lookup is folded in too (fused_embed_eh_norm); otherwise the embeds are
# precomputed and go through the model-local fused_eh_norm.
if embed_table is not None:
eh_input = fused_embed_eh_norm(
positions,
input_ids,
embed_table,
previous_hidden_states,
self.enorm.weight,
self.hnorm.weight,
self.enorm.variance_epsilon,
)
else:
assert inputs_embeds is not None
eh_input = fused_eh_norm(
positions,
inputs_embeds,
previous_hidden_states,
self.enorm.weight,
self.hnorm.weight,
self.enorm.variance_epsilon,
)
is_sequence_parallel = self.mtp_block.use_sequence_parallel
if is_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, eh_input
)
eh_input = sp_shard(eh_input)
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
if hidden_states is None:
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
# Recycle the POST-final-norm hidden into the next draft step. The
# residual-add is fused into the final RMSNorm so it is computed
# exactly once, and the result is returned for both tuple positions:
# the draft-logits hidden (compute_logits applies the LM head only) and
# the recycled previous_hidden_states. Recycling the pre-final-norm
# hidden mismatches the draft model's hnorm and lowers MTP acceptance;
# post-norm recycle matches deepseek_mtp.py (PR #45895). The tuple form
# is understood by both the V2 speculator (isinstance-tuple check) and
# the legacy proposer (model_returns_tuple is True for the
# DeepSeekMTPModel architecture).
if is_sequence_parallel:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
else:
# The MoE output is left un-reduced; fuse its all-reduce into the
# final norm, as the main model does at layer boundaries.
hidden_states, _ = fused_allreduce_rms_norm(
hidden_states, residual, self.shared_head.norm
)
return hidden_states, hidden_states
class DeepseekV32MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
self.layers = torch.nn.ModuleDict(
{
str(idx): DeepseekV32MultiTokenPredictorLayer(
vllm_config, f"{prefix}.layers.{idx}"
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.embed_tokens = make_input_embedding(
config.vocab_size,
config.hidden_size,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "embed_tokens"),
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
)
# A full on-rank table lets the eh_norm fusion fold in the embedding gather.
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
self.logits_processor = LogitsProcessor(config.vocab_size)
def set_skip_topk(self, skip: bool):
# index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse.
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "skip_topk"):
self_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor):
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
num_slots = slot_ids.numel()
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
# With a replicated table, defer the embedding gather to fused_eh_norm
# (folded into the enorm/hnorm/cat launch); otherwise gather it here.
embed_table = None
if inputs_embeds is None:
if self.replicated_embed:
embed_table = self.embed_tokens.weight
else:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
embed_table,
current_step_idx,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# hidden_states is already post-final-norm (produced in the layer
# forward and recycled as-is); apply the LM head only, without a
# second RMSNorm.
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
"""Greedy draft token ids via per-rank argmax over the vocab shard.
Saves the full-vocab all-gather ``compute_logits`` does; same tokens.
Name is fixed by the protocol the proposer probes for
(``use_local_argmax_reduction``).
"""
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
return self.logits_processor.get_top_tokens(
mtp_layer.shared_head.head, hidden_states
)
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
self.model = DeepseekV32MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
self.set_moe_parameters()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], self.config.hidden_size
)
def set_moe_parameters(self):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers = []
self.moe_mlp_layers = []
example_moe = None
for layer in self.model.layers.values():
mlp = layer.mtp_block.mlp
if isinstance(mlp, DeepseekV2MoE):
example_moe = mlp
self.moe_mlp_layers.append(mlp)
self.moe_layers.append(mlp.experts)
self.extract_moe_parameters(example_moe)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
return self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
"""See ``DeepseekV32MultiTokenPredictor.get_top_tokens``."""
return self.model.get_top_tokens(hidden_states, spec_step_idx)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"eh_proj",
"shared_head",
]
shared_weight_names = ["embed_tokens"]
spec_layer_weight = False
shared_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
if weight_name in shared_weight_names:
shared_weight = True
break
if not spec_layer_weight:
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
elif shared_weight:
name = name.replace(f"model.layers.{spec_layer}.", "model.")
return name
def is_unused_checkpoint_weight(self, name: str) -> bool:
return (
get_spec_layer_idx_from_weight_name(self.config, name) is None
and "embed_tokens" not in name
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
("fused_qkv_a_proj", "q_a_proj", 0),
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
("wk_weights_proj", "wk", 0),
("wk_weights_proj", "weights_proj", 1),
]
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_pending_wk_fp8: dict = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
continue
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
# A tied top-level embed_tokens has no spec layer to rewrite
# from; the draft needs its own copy under PP.
param = params_dict.get(name) if "embed_tokens" in name else None
if param is not None:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
name = self._rewrite_spec_layer_name(spec_layer, name)
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
_pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
) and name_mapped not in params_dict:
continue
else:
name = name_mapped
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
num_chunks = 1
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in chunk_name:
continue
is_expert_weight = True
name_mapped = chunk_name.replace(weight_name, param_name)
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if (
spec_layer != self.model.mtp_start_layer_idx
and ".layers" not in name
):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
loaded_layers: set[int] = set()
for param_name in loaded_params:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
if spec_layer is not None:
loaded_layers.add(spec_layer)
for layer_idx in range(
self.model.mtp_start_layer_idx,
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
):
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
raise ValueError(
f"MTP speculative decoding layer {layer_idx} weights "
f"missing from checkpoint."
)
return loaded_params
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,555 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
from vllm.cute_utils import _TORCH_TO_CUTE_DTYPE, cvt
from vllm.platforms import current_platform
def _make_fake_tensor(dtype, shape, divisibility):
stride = tuple(
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
for i in range(len(shape))
)
return cute.runtime.make_fake_tensor(
dtype,
shape,
stride,
assumed_align=divisibility * dtype.width // 8,
)
def is_fused_q_cutedsl_supported(
q_pe: torch.Tensor,
index_q: torch.Tensor | None,
ql_nope: torch.Tensor,
*,
has_indexer: bool,
quantize_mqa: bool,
) -> bool:
if not (
current_platform.has_device_capability(100)
and q_pe.dtype == torch.bfloat16
and q_pe.shape[-1] == 64
and (
not quantize_mqa
or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512)
)
# One warp per head group; a non-multiple would trip the kernel's own
# assert instead of falling back to Triton.
and q_pe.shape[1] % 4 == 0
):
return False
return not has_indexer or (
index_q is not None
and index_q.dtype == torch.bfloat16
and index_q.shape[1] % 16 == 0
and index_q.shape[-1] == 128
)
def fused_q_cutedsl(
positions: torch.Tensor,
q_pe: torch.Tensor,
rope_cache: torch.Tensor,
ql_nope: torch.Tensor,
q_scale: torch.Tensor,
mqa_output: torch.Tensor,
idx_q: torch.Tensor,
idx_rope_cache: torch.Tensor,
idx_weights: torch.Tensor,
idx_weights_softmax_scale: float,
idx_weights_head_scale: float,
idx_q_fp8: torch.Tensor,
idx_weights_out: torch.Tensor,
has_indexer: bool = True,
index_rope_interleave: bool = True,
quantize_mqa: bool = True,
) -> None:
_, num_heads, rope_dim = q_pe.shape
nope_dim = ql_nope.shape[-1] if quantize_mqa else 0
_, num_idx_heads, idx_dim = idx_q.shape
if has_indexer:
idx_rope_type = _TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
idx_weights_type = _TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
else:
idx_dim = num_idx_heads = 0
idx_q = idx_rope_cache = idx_q_fp8 = None
idx_weights = idx_weights_out = None
idx_rope_type = idx_weights_type = None
rope_type = _TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
compiled = FusedQKernel.compile(
rope_dim,
nope_dim,
num_heads,
rope_type,
idx_dim,
num_idx_heads,
idx_rope_type,
idx_weights_type,
index_rope_interleave,
)
compiled(
positions,
q_pe,
rope_cache,
ql_nope if quantize_mqa else None,
q_scale.view(1) if quantize_mqa else None,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
float(idx_weights_softmax_scale * idx_weights_head_scale),
)
class FusedQKernel:
def __init__(
self,
rope_dim: int,
nope_dim: int,
num_heads: int,
idx_dim: int,
num_idx_heads: int,
index_rope_interleave: bool,
) -> None:
assert rope_dim == 64
assert nope_dim in (0, 512)
assert idx_dim in (128, 0)
self.rope_dim = rope_dim
self.nope_dim = nope_dim
self.num_heads = num_heads
self.idx_dim = idx_dim
self.num_idx_heads = num_idx_heads
self.index_rope_interleave = index_rope_interleave
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
# indexer: rope_dim=64, nope_dim=64, num_heads=32
self.num_warps = 4
assert num_heads % self.num_warps == 0
assert num_idx_heads % (4 * self.num_warps) == 0
self.num_ctas_per_tok = num_heads // self.num_warps
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
@cute.jit
def __call__(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
stream: CUstream,
):
num_tokens = positions.shape[0]
if cutlass.const_expr(self.idx_dim == 0):
grid = (num_tokens, self.num_ctas_per_tok, 1)
else:
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
self.kernel(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
).launch(
grid=grid,
block=(self.num_warps * 32, 1, 1),
stream=stream,
use_pdl=True,
)
@cute.kernel
def kernel(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
):
if cutlass.const_expr(self.idx_dim == 0):
token_id, group_id, _ = cute.arch.block_idx()
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
token_id,
group_id,
)
else:
# CTA-specialization
bid, _, _ = cute.arch.block_idx()
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
if bid < num_mqa_ctas:
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
bid // self.num_ctas_per_tok,
bid % self.num_ctas_per_tok,
)
else:
bid -= num_mqa_ctas
self.indexer(
positions,
idx_q,
idx_q_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
bid // self.num_ctas_per_idx_tok,
bid % self.num_ctas_per_idx_tok,
)
@cute.jit
def mqa(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
head_id = group_id * self.num_warps + warp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
# no need to do Q calculation for MHA
if cutlass.const_expr(self.nope_dim == 0):
q0, q1 = cvt.bf16x2_to_fp32x2(
cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id]
)
cos = q_pe_rope_cache[pos, lane_id].to(Float32)
sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32)
result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin)
cute.arch.griddepcontrol_launch_dependents()
cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result
return
inv_scale = 1.0 / q_scale[0]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
##### issue all loads asap #####
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
src_ql_nope = cute.local_tile(
ql_nope[token_id, head_id, None], (16,), (lane_id,)
)
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
##### process NoPE #####
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
dst_Q_nope = cute.local_tile(
mqa_output[token_id, head_id, None], (16,), (lane_id,)
)
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
##### process RoPE ######
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rCos = rCos_raw.to(Float32)
rSin = rSin_raw.to(Float32)
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
cute.arch.griddepcontrol_launch_dependents()
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
dst_Q_rope = cute.local_tile(
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
)
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
@cute.jit
def indexer(
self,
positions: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
subwarp_id = tid // 8
sublane_id = tid % 8
head_id = group_id * (4 * self.num_warps) + subwarp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
##### issue all loads first #####
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
if cutlass.const_expr(self.index_rope_interleave):
src_idx_q_rope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
else:
src_idx_q_rope = cute.zipped_divide(
idx_q[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_8B,
src_idx_q_rope[None, 0 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
)
cute.copy(
cp_8B,
src_idx_q_rope[None, 8 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
)
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
src_idx_q_nope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
# unpack to FP32
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
rCos = rCos_raw
rSin = rSin_raw
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
# apply rope
for i in cutlass.range_constexpr(4):
if cutlass.const_expr(self.index_rope_interleave):
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
rQ_rope_f32[i * 2 + 0] = r0
rQ_rope_f32[i * 2 + 1] = r1
else:
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
rQ_rope_f32[0 + i] = r0
rQ_rope_f32[4 + i] = r1
# amax
amax = Float32(1e-4)
for i in cutlass.range_constexpr(8):
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
# warp reduction among 8 lanes
for i in cutlass.range_constexpr(3):
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
amax = cute.arch.fmax(amax, other)
# compute scale from amax
# exp2(ceil(log2(scale))) via bit manipulation
scale = amax * (1.0 / 448.0)
bits = scale.bitcast(Uint32)
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
scale = exp_bits.bitcast(Float32)
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
for i in cutlass.range_constexpr(8):
rQ_nope_f32[i] *= inv_scale
rQ_rope_f32[i] *= inv_scale
cute.arch.griddepcontrol_launch_dependents()
# quantize and store
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
dst_idx_q_nope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
if cutlass.const_expr(self.index_rope_interleave):
dst_idx_q_rope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
else:
dst_idx_q_rope = cute.zipped_divide(
idx_q_fp8[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (0,)),
dst_idx_q_rope[None, 0 + sublane_id],
)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (1,)),
dst_idx_q_rope[None, 8 + sublane_id],
)
# scale indexer weights
if sublane_id == 0:
w = idx_weights[token_id, head_id].to(Float32)
idx_weights_out[token_id, head_id] = w * scale * weight_scale
@staticmethod
@cache
def compile(
rope_dim: int,
nope_dim: int,
num_heads: int,
rope_type: type[cutlass.Numeric],
idx_dim: int,
num_idx_heads: int,
idx_rope_type: type[cutlass.Numeric] | None,
idx_weights_type: type[cutlass.Numeric] | None,
index_rope_interleave: bool,
):
num_tokens = cute.sym_int()
max_pos = cute.sym_int()
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
q_pe = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
)
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
ql_nope = (
_make_fake_tensor(
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
)
if nope_dim
else None
)
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None
mqa_output = _make_fake_tensor(
Float8E4M3FN if nope_dim else BFloat16,
(num_tokens, num_heads, nope_dim + rope_dim),
divisibility=16,
)
if idx_rope_type is not None:
index_q = _make_fake_tensor(
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_rope_cache = _make_fake_tensor(
idx_rope_type, (max_pos, rope_dim), divisibility=8
)
index_weights = _make_fake_tensor(
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
)
index_q_fp8 = _make_fake_tensor(
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_weights_out = _make_fake_tensor(
Float32, (num_tokens, num_idx_heads), divisibility=4
)
else:
index_q = index_rope_cache = index_q_fp8 = None
index_weights = index_weights_out = None
kernel = FusedQKernel(
rope_dim,
nope_dim,
num_heads,
idx_dim,
num_idx_heads,
index_rope_interleave,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
index_q,
index_rope_cache,
index_weights,
index_q_fp8,
index_weights_out,
Float32(0.0),
stream,
options="--enable-tvm-ffi",
)
-56
View File
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4 model — hardware-isolated entry point.
The actual implementation lives under ``nvidia/`` and ``amd/``; this module
picks the right one for the current platform and re-exports the public
classes used by the model registry and quantization config lookup.
"""
from vllm.platforms import current_platform
from .quant_config import DeepseekV4FP8Config
# Pick the per-platform implementation. The NVIDIA branch is the static
# default that mypy sees; the ROCm/XPU branches override at runtime and are
# kept type-compatible via ``# type: ignore[assignment]``.
if current_platform.is_rocm():
from .amd.dspark import ( # type: ignore[assignment]
DSparkDeepseekV4ForCausalLM,
)
from .amd.model import DeepseekV4ForCausalLM
from .amd.mtp import DeepSeekV4MTP
from .common.vl_model import ( # type: ignore[assignment]
DeepseekV4ForConditionalGeneration,
)
elif current_platform.is_xpu():
from .vl_stub import ( # type: ignore[assignment]
DeepseekV4ForConditionalGeneration,
)
from .xpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
from .xpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
from .xpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
elif current_platform.is_cpu():
from .cpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
from .cpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
from .cpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
from .vl_stub import ( # type: ignore[assignment]
DeepseekV4ForConditionalGeneration,
)
else:
from .common.vl_model import ( # type: ignore[assignment]
DeepseekV4ForConditionalGeneration,
)
from .nvidia.dspark import ( # type: ignore[assignment]
DSparkDeepseekV4ForCausalLM,
)
from .nvidia.model import DeepseekV4ForCausalLM # type: ignore[assignment]
from .nvidia.mtp import DeepSeekV4MTP # type: ignore[assignment]
__all__ = [
"DSparkDeepseekV4ForCausalLM",
"DeepSeekV4MTP",
"DeepseekV4FP8Config",
"DeepseekV4ForCausalLM",
"DeepseekV4ForConditionalGeneration",
]
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-514
View File
@@ -1,514 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark draft model for DeepSeek-V4 on ROCm/AMD (gfx950).
ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
``amd/mtp.py``:
* import ``DeepseekV4DecoderLayer`` from the AMD ``.model`` (aiter/triton
attention + MHC CustomOp path) instead of the nvidia one;
* route the MHC head through the ``HCHeadOp`` CustomOp dispatcher (aiter /
tilelang / triton / torch) instead of calling the tilelang kernels directly,
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
or TileLang fused MHC is available; False only on the torch fallback);
* use the AITER MegaMoE path (``aiter_mega_moe``) instead of the DeepGEMM
mega-MoE path.
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
pure torch / Triton and shared with the nvidia implementation unchanged.
"""
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from .mega_moe import finalize_mega_moe_weights
from .model import (
DeepseekV4DecoderLayer,
)
logger = init_logger(__name__)
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DSparkDeepseekV4Model(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
self.hidden_size = config.hidden_size
self.hc_mult = config.hc_mult
self.hc_eps = config.hc_eps
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
# Shared with the target (aliased by the speculator's loading utility).
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.main_proj = ReplicatedLinear(
config.hidden_size * len(self.target_layer_ids),
config.hidden_size,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "main_proj"),
)
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
current_vllm_config = get_current_vllm_config()
self.layers = nn.ModuleList(
[
DeepseekV4DecoderLayer(
current_vllm_config,
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
)
for i in range(self.num_dspark_layers)
]
)
# Heads: final norm + hc_head, and the Markov + confidence heads
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32), requires_grad=False
)
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", True):
self.confidence_head = DSparkConfidenceHead(
config.hidden_size + config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "confidence_head"),
)
# MHC head CustomOp dispatcher (aiter / tilelang / triton / torch),
# replacing the direct nvidia tilelang kernel call.
self.hc_head_op = HCHeadOp()
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
"""
return self.main_norm(self.main_proj(aux_hidden_states))
@torch.inference_mode()
def precompute_and_store_context_kv(
self,
main_x: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
"""Insert the sliding-window context KV for every draft layer.
Mirrors the reference DSparkAttention: each layer derives its context KV
from the SAME projected target hidden ``main_x``, via that layer's own
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
layer's context slots.
``context_slot_mappings`` is a per-layer list (each entry is the context
slot mapping for that layer's kv-cache group, since the hybrid manager may
place draft layers in different groups). ``None`` (or a ``None`` entry)
runs the projection to reserve workspace but writes nothing (profiling).
"""
for i, layer in enumerate(self.layers):
slot_mapping = (
None if context_slot_mappings is None else context_slot_mappings[i]
)
attn = layer.attn
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
qr_kv, _ = attn.fused_wqa_wkv(main_x)
kv = qr_kv[..., attn.q_lora_rank :]
kv = attn.kv_norm(kv)
if slot_mapping is None:
continue
_insert_context_kv(attn, kv, context_positions, slot_mapping)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
residual = post_mix = res_mix = None
for layer in self.layers:
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
)
# On the fused-MHC path the trailing hc_post must be applied here; on the
# aiter unfused path (ROCm default) the decoder layer already applied
# hc_post in-layer and returned None mixes, so this is skipped. Mirrors
# amd/mtp.py.
last_layer = self.layers[-1]
if last_layer.use_fused_mhc:
hidden_states = last_layer.hc_post(
hidden_states, residual, post_mix, res_mix
)
# hc_head reduces the hc copies; return the PRE-norm head hidden.
hidden_states = self.hc_head_op(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
self.hc_head_base,
self.rms_norm_eps,
self.hc_eps,
)
return hidden_states
def _insert_context_kv(
attn: nn.Module,
kv: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
query and discard it, since context tokens have no query). Mirrors
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
"""
swa_cache = attn.swa_cache_layer.kv_cache
block_size = attn.swa_cache_layer.block_size
cos_sin_cache = attn.rotary_emb.cos_sin_cache
cache_dtype = swa_cache.dtype
n_ctx = kv.shape[0]
dummy_q = torch.zeros(
(n_ctx, attn.n_local_heads, attn.head_dim),
dtype=kv.dtype,
device=kv.device,
)
if cache_dtype == torch.uint8:
# fp8_ds_mla UE8M0 paged layout (the gfx950 aiter SWA cache path).
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
dummy_q,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin_cache,
attn.padded_heads,
attn.eps,
block_size,
)
elif cache_dtype == torch.bfloat16:
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
dummy_q,
kv,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn.eps,
block_size,
)
else: # per-tensor fp8 (torch.float8_e4m3fn)
# NOTE(rocm): unreachable on ROCm/aiter, where the SWA cache dtype is
# uint8 (fp8_ds_mla) or bfloat16. This branch relies on FlashInfer-only
# attributes (``_flashinfer_fp8_*``) that the aiter attention layer does
# not define; kept for parity with the nvidia path.
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
dummy_q,
kv,
dummy_q_fp8,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn._flashinfer_fp8_kv_scale,
attn._flashinfer_fp8_q_scale_inv,
attn.eps,
block_size,
)
class DSparkDeepseekV4ForCausalLM(nn.Module):
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
# load_dspark_model always aliases the target's.
has_own_embed_tokens = False
has_own_lm_head = False
# Full-vocab draft: draft ids are target ids, no remapping needed.
draft_id_to_target_id = None
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
self.model = DSparkDeepseekV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# Shared with the target (aliased by the speculator's load utility).
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
# --- Hooks used by the speculator -------------------------------------
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
return self.model.combine_hidden_states(aux_hidden_states)
def get_draft_kv_cache_layer_names(self) -> list[str]:
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
# layer, named by its prefix.
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
def precompute_and_store_context_kv(
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
self.model.precompute_and_store_context_kv(
context_states, context_positions, context_slot_mappings
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
# Returns the pre-norm hc_head hidden ([T, hidden_size]).
return self.model(input_ids, positions, inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Base logits U_k = lm_head(norm(head_hidden))."""
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Full-vocab draft: base logits, no d2t scatter.
return self.compute_logits(hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
return draft_ids # full-vocab: draft ids are target ids
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
if self.model.confidence_head is None:
raise RuntimeError(
"compute_confidence() requires a confidence head, but the "
"checkpoint did not provide confidence_head weights."
)
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
# --- Weight loading ----------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
Non-mtp weights (embed/head/main layers) belong to the target model and
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
routed_experts_prefix=(
"" if self.model.layers[0].ffn.use_mega_moe else "routed_experts"
),
)
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
stacked_params_mapping = [
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_confidence_head = False
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
continue
name = mapped
if "confidence_head." in name:
loaded_confidence_head = True
# ``.scale`` -> per-method scale suffix.
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
# E8M0 expert scales: keep raw exponent bytes.
if ".experts." in name:
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
success = param.weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(name_mapped)
break
continue
# Stacked rules only apply to decoder-layer weights. Head-stack params
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
param.weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
break
else:
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
)
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def process_weights_after_loading(self) -> None:
finalize_mega_moe_weights(self)
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
Returns None for non-mtp weights (owned by the target model).
"""
m = re.match(r"mtp\.(\d+)\.(.*)", name)
if m is None:
return None
stage = int(m.group(1))
rest = m.group(2)
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
return None
# Head-stack params live at model level (mtp.last), context combiner at
# model level (mtp.0); everything else is a per-layer decoder block.
head_prefixes = (
"norm.",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
"markov_head.",
"confidence_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
):
return f"model.{rest}"
return f"model.layers.{stage}.{rest}"
-301
View File
@@ -1,301 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4."""
from typing import Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
scaled_dequantize,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
@triton.jit
def _append_shared_expert_kernel(
topk_ids_ptr,
topk_weights_ptr,
out_ids_ptr,
out_weights_ptr,
num_tokens,
shared_expert_id,
TOPK: tl.constexpr,
ROUTED_PER_RANK: tl.constexpr,
BLOCK_M: tl.constexpr,
):
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
mask = rows < num_tokens
for k in tl.static_range(TOPK):
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
tl.store(
out_ids_ptr + rows * (TOPK + 1) + TOPK,
tl.zeros_like(rows) + shared_expert_id,
mask=mask,
)
tl.store(
out_weights_ptr + rows * (TOPK + 1) + TOPK,
tl.full([BLOCK_M], 1.0, tl.float32),
mask=mask,
)
class DeepseekV4AiterMegaMoEExperts(nn.Module):
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
# instance and passes its own weights per call.
_runtime_cache: dict[tuple, Any] = {}
def __init__(
self,
vllm_config: VllmConfig,
*,
num_experts: int,
top_k: int,
hidden_size: int,
intermediate_size: int,
swiglu_limit: float | None,
fuse_shared_expert: bool = False,
prefix: str = "",
):
super().__init__()
ep_group = get_ep_group()
self.ep_rank = ep_group.rank_in_group
self.ep_size = ep_group.world_size
self.num_routed_local_experts = num_experts // self.ep_size
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
# A fused shared expert is one extra local expert per rank and one
# extra route per token, to the token's own rank.
self.fuse_shared_expert = fuse_shared_expert
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
self.num_experts = self.num_local_experts * self.ep_size
self.routed_top_k = top_k
self.top_k = top_k + fuse_shared_expert
self.shared_expert_id = (
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
)
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.swiglu_limit = swiglu_limit or 0.0
self.max_num_tokens = next_power_of_2(
vllm_config.scheduler_config.max_num_batched_tokens
)
self.prefix = prefix
def make_param(rows: int, cols: int) -> nn.Parameter:
param = nn.Parameter(
torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8),
requires_grad=False,
)
set_weight_attrs(param, {"weight_loader": self.weight_loader})
return param
self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2)
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
# (max_tok_per_rank, runtime), ascending by capacity.
self._runtimes: list[tuple[int, Any]] = []
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
local_expert_id = expert_id - self.experts_start_idx
if not 0 <= local_expert_id < self.num_routed_local_experts:
return False if return_success else None
expert_data = param.data[local_expert_id]
if shard_id in ("w1", "w3"):
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size)
elif shard_id != "w2":
raise ValueError(f"Unsupported expert shard id: {shard_id}")
expert_data.copy_(loaded_weight.view(torch.uint8))
return True if return_success else None
def load_shared_expert(self, shared_experts: nn.Module) -> None:
slot = self.num_routed_local_experts
for linear, weight, weight_scale in (
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
):
block_scale = get_fp8_block_weight_scale(linear)
assert block_scale is not None, "shared expert must be block-FP8"
group = GroupShape(
linear.weight.shape[0] // block_scale.shape[0],
linear.weight.shape[1] // block_scale.shape[1],
)
packed, scale = mxfp4_quantize(
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
)
weight.data[slot].copy_(packed.view(torch.uint8))
weight_scale.data[slot].copy_(scale.view(torch.uint8))
def finalize_weights(self) -> None:
if self._runtimes:
return
n = self.num_local_experts
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
self.w13_weight.data, 16, True
)
self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True
)
self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
self.w2_weight.data, 16, False
)
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
)
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
FIXED_SLOT_MAX_MTPR,
)
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
# must also be a power of two.
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
capacities = [self.max_num_tokens]
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
fixed_slot_max_tokens
):
capacities.insert(0, fixed_slot_max_tokens)
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
build_mega_moe_bundle_plan,
)
return build_mega_moe_bundle_plan(
max_tok_per_rank,
experts_per_rank=self.num_local_experts,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
world_size=self.ep_size,
).fixed_slot_dispatch
def get_runtime(self, max_tok_per_rank: int) -> Any:
key = (
id(get_ep_group().cpu_group),
torch.accelerator.current_device_index(),
self.num_experts,
self.top_k,
self.hidden_size,
self.intermediate_size,
max_tok_per_rank,
self.swiglu_limit,
)
runtime = self._runtime_cache.get(key)
if runtime is None:
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
runtime = MegaMoEV2(
rank=self.ep_rank,
world_size=self.ep_size,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
experts=self.num_experts,
topk=self.top_k,
quant="a8w4",
w1=self.w13_weight,
w1_scale=self.w13_weight_scale,
w2=self.w2_weight,
w2_scale=self.w2_weight_scale,
max_tok_per_rank=max_tok_per_rank,
swiglu_limit=self.swiglu_limit,
)
self._runtime_cache[key] = runtime
return runtime
def select_runtime(self, num_tokens: int) -> Any:
# Dispatch is collective, so every EP rank must pick the same runtime.
if is_forward_context_available():
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is not None:
num_tokens = max(
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
)
for max_tok_per_rank, runtime in self._runtimes:
if num_tokens <= max_tok_per_rank:
return runtime
raise ValueError(
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
)
def append_shared_expert(
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Remap routed ids to the per-rank (routed + shared) layout and append
a weight-1.0 route to this rank's shared slot."""
num_tokens = topk_ids.shape[0]
out_ids = torch.empty(
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
)
out_weights = torch.empty(
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
)
if num_tokens:
block_m = 64
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
topk_ids.contiguous(),
topk_weights.to(torch.float32).contiguous(),
out_ids,
out_weights,
num_tokens,
self.shared_expert_id,
TOPK=self.routed_top_k,
ROUTED_PER_RANK=self.num_routed_local_experts,
BLOCK_M=block_m,
)
return out_weights, out_ids
def forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
assert self._runtimes
runtime = self.select_runtime(hidden_states.shape[0])
if self.fuse_shared_expert:
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
runtime._s1_w1 = self.w13_weight
runtime._s1_w1_scale = self.w13_weight_scale
runtime.w2 = self.w2_weight
runtime.w2_scale = self.w2_weight_scale
return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids)
DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
def finalize_mega_moe_weights(model: nn.Module) -> None:
for module in list(model.modules()):
experts = getattr(module, "experts", None)
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
if experts.fuse_shared_expert:
experts.load_shared_expert(module.shared_experts)
module.shared_experts = None
experts.finalize_weights()
File diff suppressed because it is too large Load Diff
-558
View File
@@ -1,558 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
pieces that have no analogue in V3/V32:
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
the fused ``eh_proj``);
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
* V4-specific checkpoint weight-name remapping in ``load_weights``.
"""
import typing
from collections.abc import Callable, Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.models.deepseek_v4.quant_config import align_mtp_quant_config
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .mega_moe import finalize_mega_moe_weights
from .model import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which MoERunner method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
# Other FP8 linear scales (including shared experts) always use
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
topk_indices_buffer: torch.Tensor,
prefix: str,
aux_stream_list: list[torch.cuda.Stream] | None = None,
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
self.rms_norm_eps = config.rms_norm_eps
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
# fusing them the way V3 does with eh_proj.
self.e_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.e_proj",
)
self.h_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.h_proj",
)
self.hc_eps = config.hc_eps
self.hc_mult = config.hc_mult
self.hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
self.mtp_block = DeepseekV4DecoderLayer(
vllm_config,
prefix,
topk_indices_buffer=topk_indices_buffer,
aux_stream_list=aux_stream_list,
)
self.hc_head_op = HCHeadOp()
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
# reshape to (T, hc_mult, D) — the training-time layout — before
# the fused norm pass so both inputs are 3D-friendly.
previous_hidden_states = previous_hidden_states.view(
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
self.enorm.weight.data,
self.hnorm.weight.data,
self.enorm.variance_epsilon,
self.hc_mult,
)
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
inputs_embeds
).unsqueeze(-2)
hidden_states, residual, post_mix, res_mix = self.mtp_block(
positions=positions, x=hidden_states, input_ids=input_ids
)
if self.mtp_block.use_fused_mhc:
hidden_states = self.mtp_block.hc_post(
hidden_states, residual, post_mix, res_mix
)
# Return the flat pre-hc_head residual so it can be re-fed as the
# next spec step's `previous_hidden_states` when
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
return hidden_states.flatten(1)
class DeepSeekV4MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
self.device = current_platform.device_type
topk_tokens = config.index_topk
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
topk_tokens,
dtype=torch.int32,
device=self.device,
)
# Three aux streams shared across all MTP layers, mirroring
# DeepseekV4Model. ROCm runs the same work serially for now.
aux_stream_list = (
None
if current_platform.is_rocm()
else [torch.cuda.Stream() for _ in range(3)]
)
# to map the exact layer index from weights
self.layers = torch.nn.ModuleDict(
{
str(idx): DeepSeekV4MultiTokenPredictorLayer(
vllm_config,
self.topk_indices_buffer,
f"{prefix}.layers.{idx}",
aux_stream_list=aux_stream_list,
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
def _pre_head(
self,
mtp_layer: DeepSeekV4MultiTokenPredictorLayer,
hidden_states: torch.Tensor,
) -> torch.Tensor:
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
# hc_head here so logits are computed from the dense hidden state.
hidden_states = hidden_states.view(
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
)
hidden_states = mtp_layer.hc_head_op(
hidden_states,
mtp_layer.hc_head_fn,
mtp_layer.hc_head_scale,
mtp_layer.hc_head_base,
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
return _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
logits = self.logits_processor(
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
)
return logits
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# Vocab-parallel argmax for the greedy draft: per-rank head projection
# + local argmax + a [batch, 2*tp] (value, index) reduce, instead of
# all-gathering the full vocab logits.
return self.logits_processor.get_top_tokens(
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
)
class DeepSeekV4MTP(nn.Module):
# The draft head reuses DeepseekV4DecoderLayer, so its fused linears must
# unfold to the same checkpoint shards for per-layer quant config lookup.
packed_modules_mapping = DeepseekV4ForCausalLM.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
align_mtp_quant_config(self.quant_config, self.config)
self.model = DeepSeekV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
# Greedy-draft path used when use_local_argmax_reduction is enabled:
# vocab-parallel argmax, no full-vocab logits.
return self.model.get_top_tokens(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Weight name remapping for checkpoint compatibility.
# Maps checkpoint weight paths to model parameter paths.
WEIGHT_NAME_REMAPPING: dict[str, str] = {
".emb.tok_emb.weight": ".embed_tokens.weight",
".head.weight": ".shared_head.head.weight",
".norm.weight": ".shared_head.norm.weight",
}
def _remap_weight_name(name: str) -> str:
"""Remap checkpoint weight names to model parameter names."""
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
if old_pattern in name:
name = name.replace(old_pattern, new_pattern)
return name
def _find_mtp_layer_idx(name: str) -> int:
subnames = name.split(".")
for subname in subnames:
try:
# we return the first encountered integer
return int(subname)
except ValueError:
continue
return 0
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _resolve_scale_name(name: str) -> str:
# Quark checkpoints and QuarkW8A8Fp8PerBlock use ``.weight_scale``.
# Native block-FP8 layers register ``.weight_scale_inv``. Rename
# to ``_inv`` only when that variant exists and the plain one
# does not.
if name.endswith(".weight_scale") and name not in params_dict:
inv = name.removesuffix(".weight_scale") + ".weight_scale_inv"
if inv in params_dict:
return inv
# Otherwise leave the name unchanged: either it already matches a
# param, or it is genuinely unknown and should surface the normal
# KeyError downstream rather than be silently rewritten.
return name
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
routed_experts_prefix=(
"" if first_layer.mtp_block.ffn.use_mega_moe else "routed_experts"
),
)
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
# for the rename below based on the model's expert dtype.
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
for name, loaded_weight in weights:
mtp_layer_idx = _find_mtp_layer_idx(name)
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
# `model.layers.{num_hidden_layers + i}.*` so that
# get_spec_layer_idx_from_weight_name can identify them.
name = name.replace(
f"mtp.{mtp_layer_idx}.",
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
)
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
name = _remap_weight_name(name)
name = self._rewrite_spec_layer_name(spec_layer, name)
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
continue
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
continue
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
name = _resolve_scale_name(name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
break
else:
if ".experts." in name:
# Reinterpret E8M0 scales as uint8 to preserve raw
# exponent bytes; numeric copy_() would zero them.
# Mirrors the main DeepseekV4 loader.
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for mapping in expert_mapping:
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
loaded_params.add(name_mapped)
break
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
)
if name.endswith(".ffn.gate.bias"):
name = name.replace(".bias", ".e_score_correction_bias")
name = _resolve_scale_name(name)
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
loaded_layers: set[int] = set()
for param_name in loaded_params:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
if spec_layer is not None:
loaded_layers.add(spec_layer)
for layer_idx in range(
self.model.mtp_start_layer_idx,
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
):
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
raise ValueError(
f"MTP speculative decoding layer {layer_idx} weights "
f"missing from checkpoint. The checkpoint may have "
f"been quantized without including the MTP layers. "
f"Use a checkpoint that includes MTP layer weights, "
f"or disable speculative decoding."
)
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def process_weights_after_loading(self) -> None:
finalize_mega_moe_weights(self)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"h_proj",
"e_proj",
"shared_head",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
]
shared_weight_names = ["embed_tokens"]
spec_layer_weight = False
shared_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
if weight_name in shared_weight_names:
shared_weight = True
break
if not spec_layer_weight:
# treat rest weights as weights for transformer layer block
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
elif shared_weight:
# treat shared weights as top level weights
name = name.replace(f"model.layers.{spec_layer}.", "model.")
return name
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,529 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Multimodal preprocessing for the DeepSeek-V4 vision variants
(DeepSeek-V4-Flash-Vision-Exp).
The image transform and sentinel-block construction are ported from the
official repository's ``image_processor.py`` so that token counts bit-match
the reference. Each ``<|deepseek_image|>`` placeholder in the prompt expands
to a variable-length block of sentinel tokens; only positions with
``type == IMAGE`` receive vision embeddings, the other sentinels are looked
up from learned embedding vectors in the model.
Unlike the reference (which uses out-of-vocab ids ``vocab_size + type``),
the sentinel block borrows five consecutive reserved tokenizer tokens
(``<|place_holder_mm_span_0431|>`` .. ``_0435|>``): they are special tokens
the tokenizer never emits from plain text, so the ids stay in-vocabulary and
work with stock token validation, logprobs and detokenization. The ids are
pure markers — every sentinel position's embedding is overwritten with
vision/sentinel vectors before the decoder layers see it, exactly like the
reference's out-of-vocab scheme.
"""
import math
from collections.abc import Mapping, Sequence
from typing import Any, assert_never
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature, DeepseekV4Config
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
MultiModalPromptUpdatesApplyResult,
PlaceholderFeaturesInfo,
UpdateMode,
_plan_prompt_updates,
)
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
COMPRESS_PAD_TO = 4
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
# Sentinel roles borrow five consecutive ``<|place_holder_mm_span_XXXX|>``
# tokens (reserved special tokens, never emitted from plain text). The order
# must match IMAGE_START..IMAGE_END above so that
# ``id == IMAGE_SENTINEL_BASE_ID + type``.
IMAGE_SENTINEL_BASE_ID = 129257
IMAGE_SENTINEL_TOKEN_NAMES = (
"<|place_holder_mm_span_0431|>", # IMAGE_START
"<|place_holder_mm_span_0432|>", # IMAGE_PAD
"<|place_holder_mm_span_0433|>", # IMAGE
"<|place_holder_mm_span_0434|>", # IMAGE_NEW_LINE
"<|place_holder_mm_span_0435|>", # IMAGE_END
)
def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor:
"""Boolean mask for image-block sentinel positions (in-vocab ids)."""
return (token_ids >= IMAGE_SENTINEL_BASE_ID) & (
token_ids < IMAGE_SENTINEL_BASE_ID + len(IMAGE_SENTINEL_TOKEN_NAMES)
)
def validate_image_sentinel_ids(tokenizer) -> None:
"""Check the borrowed sentinel ids against the tokenizer."""
for i, name in enumerate(IMAGE_SENTINEL_TOKEN_NAMES):
token_id = tokenizer.convert_tokens_to_ids(name)
if token_id != IMAGE_SENTINEL_BASE_ID + i:
raise ValueError(
f"Image sentinel token {name!r} has id {token_id}, expected "
f"{IMAGE_SENTINEL_BASE_ID + i}; the DeepSeek-V4 vision "
"sentinel block requires these consecutive reserved ids."
)
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
"""Number of LLM tokens the aligner grid occupies (N-layout, including
row/align padding)."""
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
num_tokens = n_llm_h * (n_llm_w + 1) + 2
if n_llm_h % 2 == 1:
num_tokens += n_llm_w + 1
num_tokens += (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2
return n_llm_h, n_llm_w, num_tokens
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
r = height / width
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
max_h_float = max_w_float * r
if max_w_float < 1.0:
max_w = 1
max_h = (max_n_token - 2) // (max_w + 1)
if max_h % 2 == 1:
max_h -= 1
best_width = max_w * patch_size * downsample_ratio
best_height = max_h * patch_size * downsample_ratio
elif max_h_float < 2.0:
max_h = 2
max_w = ((max_n_token - 2) // max_h) - 1
assert max_w > 1
best_width = max_w * patch_size * downsample_ratio
best_height = max_h * patch_size * downsample_ratio
else:
max_w = math.floor(max_w_float)
max_h = math.floor(max_h_float)
if max_h % 2 == 1:
max_h -= 1
beta = min(
max_w * patch_size * downsample_ratio / width,
max_h * patch_size * downsample_ratio / height,
)
best_width = math.floor(width * beta / patch_size) * patch_size
best_height = math.floor(height * beta / patch_size) * patch_size
n_llm_h, n_llm_w, num_tokens = grid_tokens(
best_height, best_width, patch_size, downsample_ratio
)
return n_llm_h, n_llm_w, best_height, best_width, num_tokens
def safe_resize(
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
):
max_n_token -= COMPRESS_PAD_TO - 1
n_llm_h, n_llm_w, num_tokens = grid_tokens(
best_height, best_width, patch_size, downsample_ratio
)
budget = max_n_token
while num_tokens > max_n_token:
n_llm_h, n_llm_w, best_height, best_width, num_tokens = solve_resize_ratio(
height, width, patch_size, downsample_ratio, budget
)
budget -= 1
return n_llm_h, n_llm_w, best_height, best_width
def load_image(
image: Image.Image,
*,
patch_size: int,
downsample_ratio: int,
max_n_token: int,
min_pixels: int,
max_wh_ratio: float | None,
):
"""Transform one PIL image into ViT patches.
Same math as the reference ``load_image``, except the image is already
decoded (vLLM supplies PIL images instead of a record dict).
"""
p = patch_size
image = image.convert("RGB")
width, height = image.size
if max_wh_ratio is not None and width > height * max_wh_ratio:
width = height * max_wh_ratio
if 0 < width * height < min_pixels:
ratio = (min_pixels / (width * height)) ** 0.5
width = int(width * ratio)
height = int(height * ratio)
best_width = math.ceil(width / p) * p
best_height = math.ceil(height / p) * p
n_llm_h, n_llm_w, best_height, best_width = safe_resize(
height, width, best_height, best_width, p, downsample_ratio, max_n_token
)
n_vit_h, n_vit_w = best_height // p, best_width // p
if max_wh_ratio is not None and image.width >= max_wh_ratio * image.height:
image = image.resize((best_width, best_height))
else:
image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))
x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255
x = ((x - 0.5) / 0.5).to(torch.bfloat16)
patches = (
x.reshape(3, n_vit_h, p, n_vit_w, p)
.permute(1, 3, 0, 2, 4)
.reshape(n_vit_h * n_vit_w, 3, p, p)
)
return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
"""Builds the N-layout token types (final order) and the aligner-row order
for IMAGE slots."""
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
pad_h = n_llm_h % 2
rows = n_llm_h + pad_h
row_len = n_llm_w + 1
pad_last = rows // 2 * row_len % 2 * 2
types = torch.tensor(
([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h
+ [IMAGE_PAD] * (row_len * pad_h),
dtype=torch.int64,
)
order = torch.arange(rows * row_len).view(rows // 2, 2, row_len)
order = order.transpose(1, 2).reshape(-1)
image_idx = torch.full((rows * row_len,), -1, dtype=torch.int64)
image_idx.view(rows, row_len)[:n_llm_h, :n_llm_w] = torch.arange(
n_llm_h * n_llm_w
).view(n_llm_h, n_llm_w)
perm = image_idx[order]
perm = perm[perm >= 0]
types = torch.cat(
[
torch.full((compress_pad,), IMAGE_PAD, dtype=torch.int64),
torch.tensor([IMAGE_START]),
types[order],
torch.full((pad_last,), IMAGE_PAD, dtype=torch.int64),
torch.tensor([IMAGE_END]),
]
)
return types, perm
def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
"""``build_image_block`` without the position-dependent compressor pad.
``start_pos`` is chosen so that ``compress_pad == 0``; the pad is instead
prepended when the block is spliced into the final prompt, where its
position is known.
"""
return build_image_block(n_llm_h, n_llm_w, COMPRESS_PAD_TO - 1)
class DeepseekV4VLImageProcessor:
"""Per-image transform (the PIL-input equivalent of the reference
``load_image``)."""
def __init__(self, config: DeepseekV4Config) -> None:
super().__init__()
self.patch_size = config.vision_patch_size
self.downsample_ratio = config.vision_downsample_ratio
self.max_n_token = config.vision_max_n_token
self.min_pixels = config.vision_min_pixels
self.max_wh_ratio = config.vision_max_wh_ratio
def __call__(self, image: Image.Image):
return load_image(
image,
patch_size=self.patch_size,
downsample_ratio=self.downsample_ratio,
max_n_token=self.max_n_token,
min_pixels=self.min_pixels,
max_wh_ratio=self.max_wh_ratio,
)
class DeepseekV4VLProcessor:
"""Minimal stand-in for the HF processor of DeepSeek-V4 vision models.
The official repository ships image preprocessing as plain functions in
``image_processor.py`` (no ``auto_map`` processor), so this class wraps
their ports directly and the model loads without ``--trust-remote-code``.
``__call__`` returns a ``BatchFeature`` with one entry per image
(flattened across images):
- ``patches``: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` bf16 ViT patches.
- ``vit_grid``: ``(num_images, 2)`` int64 ``[n_vit_h, n_vit_w]``.
- ``llm_grid``: ``(num_images, 2)`` int64 ``[n_llm_h, n_llm_w]``.
- ``perm``: concatenated per-image ``(n_llm_h * n_llm_w,)`` int64 index
selecting aligner outputs into the final N-layout order.
- ``types``: concatenated per-image pad-free sentinel block types;
``block ids = IMAGE_SENTINEL_BASE_ID + types``.
"""
def __init__(self, config: DeepseekV4Config) -> None:
super().__init__()
self.config = config
self.image_processor = DeepseekV4VLImageProcessor(config)
def __call__(
self,
text: str | None = None,
images: Sequence[Image.Image] | None = None,
return_tensors: str | None = None,
**kwargs: Any,
) -> BatchFeature:
patches_list = []
vit_grid = []
llm_grid = []
perm_list = []
types_list = []
for image in images or []:
patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = self.image_processor(image)
types, perm = build_image_block_pad_free(n_llm_h, n_llm_w)
patches_list.append(patches)
vit_grid.append((n_vit_h, n_vit_w))
llm_grid.append((n_llm_h, n_llm_w))
perm_list.append(perm)
types_list.append(types)
if not patches_list:
return BatchFeature({})
return BatchFeature(
{
"patches": torch.cat(patches_list),
"vit_grid": torch.tensor(vit_grid, dtype=torch.int64),
"llm_grid": torch.tensor(llm_grid, dtype=torch.int64),
"perm": torch.cat(perm_list),
"types": torch.cat(types_list),
}
)
class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
def get_hf_config(self) -> DeepseekV4Config:
return self.ctx.get_hf_config(DeepseekV4Config)
def get_hf_processor(self, **kwargs: object) -> DeepseekV4VLProcessor:
if kwargs:
raise ValueError(f"Unexpected processor kwargs: {sorted(kwargs)}")
return DeepseekV4VLProcessor(self.get_hf_config())
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
# ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of
# vision_max_n_token for the compressor-alignment pad, so the full
# sentinel block is bounded by vision_max_n_token; the margin is kept
# in case that reservation changes.
return {
"image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1,
}
def get_image_placeholder_token_id(self) -> int:
token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER)
if token_id is None:
raise ValueError(f"Token not found in tokenizer: {IMAGE_PLACEHOLDER}")
return token_id
def get_image_size_with_most_features(self) -> ImageSize:
hf_config = self.get_hf_config()
patch_size = hf_config.vision_patch_size
downsample_ratio = hf_config.vision_downsample_ratio
max_wh_ratio = hf_config.vision_max_wh_ratio
# Every aligner row costs an extra IMAGE_NEW_LINE token, so height is
# more expensive than width and a square is *not* optimal: search the
# LLM grids that fit the budget for the one with the most ViT patches
# (ties broken by the longer sentinel block).
budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
step = patch_size * downsample_ratio
best_grid = (1, 1)
best_key = (0, 0)
for n_llm_h in range(1, budget):
if (
grid_tokens(n_llm_h * step, step, patch_size, downsample_ratio)[2]
> budget
):
break
for n_llm_w in range(1, budget):
if max_wh_ratio is not None and n_llm_w > n_llm_h * max_wh_ratio:
break
num_tokens = grid_tokens(
n_llm_h * step, n_llm_w * step, patch_size, downsample_ratio
)[2]
if num_tokens > budget:
break
key = (n_llm_h * n_llm_w, num_tokens)
if key > best_key:
best_key, best_grid = key, (n_llm_h, n_llm_w)
n_llm_h, n_llm_w = best_grid
return ImageSize(width=n_llm_w * step, height=n_llm_h * step)
class DeepseekV4VLDummyInputsBuilder(
BaseDummyInputsBuilder[DeepseekV4VLProcessingInfo]
):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return IMAGE_PLACEHOLDER * mm_counts.get("image", 0)
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
class DeepseekV4VLMultiModalProcessor(
BaseMultiModalProcessor[DeepseekV4VLProcessingInfo]
):
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
vit_grid = hf_inputs.get("vit_grid")
llm_grid = hf_inputs.get("llm_grid")
if vit_grid is None or llm_grid is None:
empty = torch.empty(0, dtype=torch.long)
patch_sizes = perm_sizes = types_sizes = empty
else:
patch_sizes = vit_grid.prod(-1)
perm_sizes = llm_grid.prod(-1)
n_llm_h, n_llm_w = llm_grid[:, 0], llm_grid[:, 1]
# Pad-free block length; same formula as ``grid_tokens`` given
# the LLM grid.
types_sizes = (
n_llm_h * (n_llm_w + 1)
+ 2
+ (n_llm_h % 2) * (n_llm_w + 1)
+ (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2
)
return {
"patches": MultiModalFieldConfig.flat_from_sizes("image", patch_sizes),
"vit_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
"llm_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
"perm": MultiModalFieldConfig.flat_from_sizes("image", perm_sizes),
"types": MultiModalFieldConfig.flat_from_sizes(
"image", types_sizes, keep_on_cpu=True
),
}
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
image_token_id = self.info.get_image_placeholder_token_id()
validate_image_sentinel_ids(self.info.get_tokenizer())
image_embed_id = IMAGE_SENTINEL_BASE_ID + IMAGE
def get_image_replacement(item_idx: int) -> PromptUpdateDetails:
types: torch.Tensor = out_mm_kwargs["image"][item_idx]["types"].data
full = (IMAGE_SENTINEL_BASE_ID + types).tolist()
return PromptUpdateDetails.select_token_id(full, image_embed_id)
return [
PromptReplacement(
modality="image",
target=[image_token_id],
replacement=get_image_replacement,
),
]
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
# Same as the base implementation, except that each image block gets
# its compressor-alignment pad (``3 - start_pos % 4`` IMAGE_PAD
# sentinels) prepended while splicing: the pad depends on the block's
# final position in the prompt, which is unknown when the (cacheable)
# prompt updates are built.
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
pad_id = IMAGE_SENTINEL_BASE_ID + IMAGE_PAD
new_token_ids = list[int]()
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif update.mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(update.mode)
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
start_idx = len(new_token_ids)
tokens = list(update.content.full)
if tokens and update.modality == "image":
compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO
tokens = [pad_id] * compress_pad + tokens
if tokens:
content_is_embed = update.content.is_embed
is_embed = (
content_is_embed(tokens) if content_is_embed is not None else None
)
placeholders[update.modality].append(
PlaceholderFeaturesInfo(
modality=update.modality,
item_idx=update.item_idx,
start_idx=start_idx,
tokens=tokens,
is_embed=is_embed,
)
)
new_token_ids.extend(tokens)
prev_end_idx = match.end_idx
new_token_ids.extend(token_ids[prev_end_idx:])
return new_token_ids, result, placeholders
@@ -1,23 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .cache_utils import (
build_flashinfer_mixed_sparse_indices,
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
from .fused_indexer_q import MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant
from .fused_inv_rope_fp8_quant import fused_inv_rope_fp8_quant
__all__ = [
"MXFP4_BLOCK_SIZE",
"build_flashinfer_mixed_sparse_indices",
"combine_topk_swa_indices",
"compute_global_topk_indices_and_lens",
"dequantize_and_gather_k_cache",
"fused_indexer_q_rope_quant",
"fused_inv_rope_fp8_quant",
"quantize_and_insert_k_cache",
]
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,733 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
# MXFP4: 32 elements per block, packed 2 nibbles per byte, ue8m0 block scale.
MXFP4_BLOCK_SIZE = 32
@triton.jit
def _get_cos_sin(
cos_sin_cache_ptr,
cos_sin_cache_stride,
pos,
HALF_ROT_DIM: tl.constexpr,
):
block = tl.arange(0, HALF_ROT_DIM)
cos = tl.load(cos_sin_cache_ptr + pos * cos_sin_cache_stride + block)
cos = cos.to(tl.float32)
sin = tl.load(cos_sin_cache_ptr + pos * cos_sin_cache_stride + block + HALF_ROT_DIM)
sin = sin.to(tl.float32)
return cos, sin
@triton.jit
def _fp32x2_to_fp4x2(x_lo, x_hi):
# NOTE: $1 is high nibble, $2 is low nibble
return tl.inline_asm_elementwise(
"""
{
.reg .b8 tmp;
cvt.rn.satfinite.e2m1x2.f32 tmp, $1, $2;
cvt.u32.u8 $0, tmp;
}
""",
constraints="=r,f,f",
args=[x_hi, x_lo],
dtype=tl.uint32,
is_pure=True,
pack=1,
).to(tl.uint8)
@triton.jit
def _quantize_mxfp4_pair(x_lo, x_hi):
"""Quantize a block of MXFP4_BLOCK_SIZE fp32 values given as two
interleaved halves (x_lo = values at even positions in the block,
x_hi = values at odd positions). Returns:
- packed : uint8[BLOCK/2] (low nibble = quant(x_lo), high = quant(x_hi))
- ue8m0 : scalar uint8 (block scale = 2^(ue8m0 - 127))
"""
amax = tl.maximum(tl.max(tl.abs(x_lo)), tl.max(tl.abs(x_hi)))
# 6 * 2^-126 is from https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/inference/kernel.py#L163
amax = tl.maximum(amax, 6.0 * (2**-126))
# ue8m0 block scale: 2^ceil(log2(amax/6.0)).
log2_ratio = tl.math.ceil(tl.math.log2(amax * (1.0 / 6.0)))
log2_ratio = tl.minimum(tl.maximum(log2_ratio, -127.0), 127.0)
scale = tl.math.exp2(log2_ratio)
ue8m0 = (log2_ratio + 127.0).to(tl.uint8)
inv_scale = 1.0 / scale
packed = _fp32x2_to_fp4x2(x_lo * inv_scale, x_hi * inv_scale)
return packed, ue8m0
class FusedIndexerQRopeQuantTritonKernel(
VllmTritonJitKernel["FusedIndexerQRopeQuantTritonKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
num_heads: int
index_q_half_rot_dim: int
index_q_head_dim: int
fp8_max: float
use_fnuz: bool
use_explicit_fma: bool
@staticmethod
@triton.jit
def kernel(
pos_ptr,
# Index Q RoPE
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# Index Q Quantize
index_q_fp8_ptr,
index_q_fp8_stride0,
index_q_fp8_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
# Index weights
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
FP8_MAX: tl.constexpr = 448.0,
USE_FNUZ: tl.constexpr = False,
USE_EXPLICIT_FMA: tl.constexpr = False,
):
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
# through unchanged.
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
pos = tl.load(pos_ptr + tok_idx)
cos, sin = _get_cos_sin(
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
pos,
INDEX_Q_HALF_ROT_DIM,
)
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
rot_base = base_ptr + INDEX_Q_NOPE_DIM
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
if USE_EXPLICIT_FMA:
# Match HIP rotary_embedding contraction before bf16 materialization.
r_even = tl.fma(x_even, cos, -(x_odd * sin))
r_odd = tl.fma(x_odd, cos, x_even * sin)
else:
r_even = x_even * cos - x_odd * sin
r_odd = x_odd * cos + x_even * sin
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
r_even = r_even.to(tl.bfloat16).to(tl.float32)
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
if INDEX_Q_NOPE_DIM > 0:
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
# (e4m3fn) elsewhere -- matches the K cache.
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
fp8_base_ptr = (
index_q_fp8_ptr
+ tok_idx * index_q_fp8_stride0
+ head_idx * index_q_fp8_stride1
)
if INDEX_Q_NOPE_DIM > 0:
tl.store(
fp8_base_ptr + nope_offset,
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
)
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
tl.store(
fp8_rot_base + half_offset * 2,
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
)
tl.store(
fp8_rot_base + half_offset * 2 + 1,
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
)
# FP8 weight-fold contract:
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
# The per-token-per-head q_scale (fp32) IS folded into the output weights
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use
# `weights` to apply per-token Q scale inline. See the MXFP4 kernel below for
# contrasting convention (scales live with the Q values, weights are NOT
# q-scaled).
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
)
index_weights = index_weights.to(tl.float32)
index_weights *= index_q_scale
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
num_heads: int,
head_dim: int,
rope_dim: int,
use_fnuz: bool,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
num_heads=num_heads,
index_q_half_rot_dim=rope_dim // 2,
index_q_head_dim=head_dim,
fp8_max=224.0 if use_fnuz else 448.0,
use_fnuz=use_fnuz,
use_explicit_fma=current_platform.is_rocm(),
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
num_heads=num_heads,
head_dim=head_dim,
rope_dim=rope_dim,
use_fnuz=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
return dict(
positions=TritonWarmupTensor(torch.int64),
index_q=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_q_cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.index_q_half_rot_dim * 2),
),
index_weights=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads),
),
index_weights_softmax_scale=1.0,
index_weights_head_scale=1.0,
index_q_fp8=TritonWarmupTensor(
current_platform.fp8_dtype(),
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_weights_out=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.num_heads),
),
fp8_max=compile_key.fp8_max,
use_fnuz=compile_key.use_fnuz,
)
@kernel_launcher
def __call__(
self,
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
index_q_fp8: torch.Tensor,
index_weights_out: torch.Tensor,
*,
fp8_max: float,
use_fnuz: bool,
) -> LaunchSpec:
num_tokens = positions.shape[0]
num_index_q_heads = index_q.shape[1]
return (num_tokens, num_index_q_heads), dict(
pos_ptr=positions,
index_q_stride0=index_q.stride(0),
index_q_stride1=index_q.stride(1),
index_q_cos_sin_ptr=index_q_cos_sin_cache,
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
index_q_fp8_stride0=index_q_fp8.stride(0),
index_q_fp8_stride1=index_q_fp8.stride(1),
INDEX_Q_HEAD_DIM=index_q.shape[2],
index_weights_stride=index_weights.stride(0),
index_weights_out_stride=index_weights_out.stride(0),
FP8_MAX=fp8_max,
USE_FNUZ=use_fnuz,
USE_EXPLICIT_FMA=current_platform.is_rocm(),
enable_fp_fusion=True,
num_warps=1,
)
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
(`SparseMQAIndexer`) is enabled."""
if vllm_config.attention_config.indexer_sparse_logits:
return (torch.float32, torch.bfloat16)
return (torch.float32,)
class FusedIndexerQRopeMxFp4TritonKernel(
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
weights_out_dtype: torch.dtype
num_heads: int
index_q_half_rot_dim: int
index_q_head_dim: int
mxfp4_block: int
@staticmethod
@triton.jit
def kernel(
pos_ptr,
# Index Q RoPE input (fp/bf16)
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# MXFP4 Q outputs
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
index_q_mxfp4_stride0,
index_q_mxfp4_stride1,
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
index_q_scale_stride0,
index_q_scale_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
MXFP4_BLOCK: tl.constexpr,
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
# with the Q values in the output scale tensor).
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
):
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
tl.static_assert(MXFP4_BLOCK % 2 == 0)
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
pos = tl.load(pos_ptr + tok_idx)
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
out_base = (
index_q_mxfp4_ptr
+ tok_idx * index_q_mxfp4_stride0
+ head_idx * index_q_mxfp4_stride1
)
scale_base = (
index_q_scale_ptr
+ tok_idx * index_q_scale_stride0
+ head_idx * index_q_scale_stride1
)
half_off = tl.arange(0, HALF_BLOCK)
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
for b in tl.static_range(NUM_NOPE_BLOCKS):
base = b * MXFP4_BLOCK
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
tl.store(out_base + base // 2 + half_off, packed)
tl.store(scale_base + b, ue8m0)
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
rot_q_base = q_base + INDEX_Q_NOPE_DIM
for b in tl.static_range(NUM_ROPE_BLOCKS):
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
cos_b = tl.load(
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
).to(tl.float32)
sin_b = tl.load(
index_q_cos_sin_ptr
+ pos * index_q_cos_sin_stride
+ pair_off
+ INDEX_Q_HALF_ROT_DIM
).to(tl.float32)
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
r_even = x_even * cos_b - x_odd * sin_b
r_odd = x_odd * cos_b + x_even * sin_b
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
r_even = r_even.to(tl.bfloat16).to(tl.float32)
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
tl.store(out_base + rope_byte_off + half_off, packed)
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
# MXFP4 weight-fold contract:
# index_weights_out = index_weights * softmax_scale * head_scale
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
# per-block scale is applied by the downstream MXFP4 logits kernel when
# dequantizing Q — there is no per-token scalar to fold into `weights`.
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
).to(tl.float32)
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
weights_out_dtype: torch.dtype,
num_heads: int,
head_dim: int,
rope_dim: int,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
weights_out_dtype=weights_out_dtype,
num_heads=num_heads,
index_q_half_rot_dim=rope_dim // 2,
index_q_head_dim=head_dim,
mxfp4_block=MXFP4_BLOCK_SIZE,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
weights_out_dtype=_indexer_weights_out_dtypes(vllm_config),
num_heads=num_heads,
head_dim=head_dim,
rope_dim=rope_dim,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
packed_head_dim = compile_key.index_q_head_dim // 2
scale_head_dim = compile_key.index_q_head_dim // compile_key.mxfp4_block
return dict(
positions=TritonWarmupTensor(torch.int64),
index_q=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_q_cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.index_q_half_rot_dim * 2),
),
index_weights=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads),
),
index_weights_softmax_scale=1.0,
index_weights_head_scale=1.0,
index_q_packed=TritonWarmupTensor(
torch.uint8,
shape=(1, compile_key.num_heads, packed_head_dim),
),
index_q_scale=TritonWarmupTensor(
torch.uint8,
shape=(1, compile_key.num_heads, scale_head_dim),
),
index_weights_out=TritonWarmupTensor(
compile_key.weights_out_dtype,
shape=(1, compile_key.num_heads),
),
)
@kernel_launcher
def __call__(
self,
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
index_q_packed: torch.Tensor,
index_q_scale: torch.Tensor,
index_weights_out: torch.Tensor,
) -> LaunchSpec:
num_tokens = positions.shape[0]
num_index_q_heads = index_q.shape[1]
return (num_tokens, num_index_q_heads), dict(
pos_ptr=positions,
index_q_stride0=index_q.stride(0),
index_q_stride1=index_q.stride(1),
index_q_cos_sin_ptr=index_q_cos_sin_cache,
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
index_q_mxfp4_ptr=index_q_packed,
index_q_mxfp4_stride0=index_q_packed.stride(0),
index_q_mxfp4_stride1=index_q_packed.stride(1),
index_q_scale_stride0=index_q_scale.stride(0),
index_q_scale_stride1=index_q_scale.stride(1),
INDEX_Q_HEAD_DIM=index_q.shape[2],
MXFP4_BLOCK=MXFP4_BLOCK_SIZE,
index_weights_stride=index_weights.stride(0),
index_weights_out_stride=index_weights_out.stride(0),
enable_fp_fusion=True,
num_warps=1,
)
def fused_indexer_q_rope_quant(
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
# Index weights
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
use_fp4: bool = False,
weights_out_dtype: torch.dtype = torch.float32,
) -> tuple[
torch.Tensor | tuple[torch.Tensor, torch.Tensor],
torch.Tensor,
]:
"""Fused RoPE + quantize Q for the sparse indexer.
``weights_out_dtype`` is the dtype the downstream scoring kernel takes:
fp32 for the dense MQA-logits kernels, bf16 for DeepGEMM's sparse
MQA-logits kernels (CUDA MXFP4 path only).
Weight-fold semantics (important — the two paths differ):
FP8 path (use_fp4=False, default):
q_fp8 : (T, H, HEAD_DIM) platform fp8 (e4m3fnuz on gfx942,
e4m3fn elsewhere); per-token-per-head scalar scale
(NOT stored — folded into weights below)
weights_out = weights * q_scale * softmax_scale * head_scale
Rationale: a single per-token q_scale is a scalar the downstream FP8
logits kernel would otherwise multiply in. Folding it into `weights`
avoids emitting a separate tensor and is free for the logits kernel.
MXFP4 path (use_fp4=True):
q_packed : (T, H, HEAD_DIM // 2) uint8 (2 E2M1 nibbles per byte)
q_scale : (T, H, HEAD_DIM // MXFP4_BLOCK_SIZE) uint8 ue8m0 bytes
weights_out = weights * softmax_scale * head_scale
Rationale: MXFP4 has PER-BLOCK (32-element) scales that live with
the Q values — they cannot be folded into a per-token weight
scalar, so `weights` carries only the softmax and head scales.
Returns (q_quant, weights_out) where q_quant is either a Tensor (FP8) or
a (values, scales) tuple (MXFP4). This matches the union type accepted
by `SparseAttnIndexer.forward_*`.
"""
assert positions.ndim == 1
assert index_q.ndim == 3
assert index_q_cos_sin_cache.ndim == 2
num_tokens = positions.shape[0]
num_index_q_heads = index_q.shape[1]
index_q_head_dim = index_q.shape[2]
index_weights_out = torch.empty_like(index_weights, dtype=weights_out_dtype)
if use_fp4:
assert index_q_head_dim % MXFP4_BLOCK_SIZE == 0, (
f"head_dim={index_q_head_dim} must be a multiple of MXFP4 block "
f"size {MXFP4_BLOCK_SIZE}"
)
num_scale_blocks = index_q_head_dim // MXFP4_BLOCK_SIZE
index_q_packed = torch.empty(
(num_tokens, num_index_q_heads, index_q_head_dim // 2),
dtype=torch.uint8,
device=index_q.device,
)
index_q_scale = torch.empty(
(num_tokens, num_index_q_heads, num_scale_blocks),
dtype=torch.uint8,
device=index_q.device,
)
if has_cutedsl():
# lazily import, otherwise some tests fail due to CUDA driver init failure.
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
_INDEXER_Q_MXFP4_KERNEL,
)
_INDEXER_Q_MXFP4_KERNEL(
positions=positions,
q=index_q,
cos_sin_cache=index_q_cos_sin_cache,
weights=index_weights,
weights_softmax_scale=index_weights_softmax_scale,
weights_head_scale=index_weights_head_scale,
q_packed=index_q_packed,
q_scale=index_q_scale,
weights_out=index_weights_out,
)
elif current_platform.is_xpu():
assert weights_out_dtype == torch.float32, weights_out_dtype
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_mxfp4(
index_q,
positions,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_packed,
index_q_scale,
index_weights_out,
)
else:
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL(
positions,
index_q,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_packed,
index_q_scale,
index_weights_out,
)
# Values stay uint8 (2 E2M1 nibbles per byte). Scales are 4 ue8m0
# bytes per (token, head) reinterpreted as one int32, then squeezed
# from (T, H, 1) to (T, H) to match DeepGEMM's expected q_sf rank
# (prefill wants 2-D (seq_len, num_heads); decode reshapes this to
# 3-D (batch, next_n, num_heads)).
return (
index_q_packed,
index_q_scale.view(torch.int32).squeeze(-1),
), index_weights_out
assert weights_out_dtype == torch.float32, weights_out_dtype
fp8_dtype = current_platform.fp8_dtype()
use_fnuz = fp8_dtype == torch.float8_e4m3fnuz
fp8_max = 224.0 if use_fnuz else 448.0
index_q_fp8 = torch.empty_like(index_q, dtype=fp8_dtype)
if has_cutedsl():
# lazily import, otherwise some tests fail due to CUDA driver init failure.
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
_INDEXER_Q_FP8_KERNEL,
)
_INDEXER_Q_FP8_KERNEL(
positions=positions,
q=index_q,
cos_sin_cache=index_q_cos_sin_cache,
weights=index_weights,
weights_softmax_scale=index_weights_softmax_scale,
weights_head_scale=index_weights_head_scale,
q_fp8=index_q_fp8.view(torch.uint8),
weights_out=index_weights_out,
)
elif current_platform.is_xpu():
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_fp8(
index_q,
positions,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
)
elif current_platform.is_cpu():
from vllm._custom_ops import fused_indexer_q_rope_quant_cpu
fused_indexer_q_rope_quant_cpu(
positions.contiguous(),
index_q.to(torch.float32),
index_q_cos_sin_cache.to(torch.float32).contiguous(),
index_q_fp8,
index_weights.to(torch.float32),
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out,
)
else:
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
positions,
index_q,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
fp8_max=fp8_max,
use_fnuz=use_fnuz,
)
return index_q_fp8, index_weights_out
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL = FusedIndexerQRopeQuantTritonKernel()
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL = FusedIndexerQRopeMxFp4TritonKernel()
@@ -1,538 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
"""
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
class FusedInvRopeFP8QuantKernel(
VllmTritonJitKernel["FusedInvRopeFP8QuantKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
heads_per_group: int
fp8_max: float
quant_group_size: int
chunks_per_head: int
nope_dim: int
half_rope: int
tma_aligned_scales: bool
launch_pdl: bool
quantize: bool
@staticmethod
# scale_stride_k = align(num_tokens, 4) is only 4-aligned, so its Triton int
# class varies with batch size; not specialized so one warmup key covers all
# (int64-cast, so perf-neutral).
@triton.jit(do_not_specialize=["num_tokens", "scale_stride_k"])
def kernel(
o_ptr,
positions_ptr,
cos_sin_cache_ptr,
out_ptr,
scale_ptr,
num_tokens,
heads_per_group: tl.constexpr,
o_stride_token,
o_stride_head,
cache_stride_pos,
out_stride_group,
out_stride_token,
scale_stride_group,
scale_stride_k,
fp8_max: tl.constexpr,
eps: tl.constexpr,
QUANT_GROUP_SIZE: tl.constexpr,
CHUNKS_PER_HEAD: tl.constexpr,
NOPE_DIM: tl.constexpr,
HALF_ROPE: tl.constexpr,
QUANTIZE: tl.constexpr,
TMA_ALIGNED_SCALES: tl.constexpr,
launch_pdl: tl.constexpr,
):
# Cast every stride to int64 — without this, Python-int strides are
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
pid_token = tl.program_id(0).to(tl.int64)
pid_gh = tl.program_id(1).to(tl.int64)
o_stride_token = o_stride_token.to(tl.int64)
o_stride_head = o_stride_head.to(tl.int64)
cache_stride_pos = cache_stride_pos.to(tl.int64)
out_stride_group = out_stride_group.to(tl.int64)
out_stride_token = out_stride_token.to(tl.int64)
scale_stride_group = scale_stride_group.to(tl.int64)
scale_stride_k = scale_stride_k.to(tl.int64)
g = pid_gh // heads_per_group
head_in_group = pid_gh % heads_per_group
global_head = pid_gh
qb_start = head_in_group * CHUNKS_PER_HEAD
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.extra.cuda.gdc_wait()
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
if pid_token >= num_tokens:
if not QUANTIZE:
return
if TMA_ALIGNED_SCALES:
packed_offsets = tl.arange(0, CHUNKS_PER_HEAD // 4)
scale_addr = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ (head_in_group * (CHUNKS_PER_HEAD // 4) + packed_offsets)
* scale_stride_k
)
tl.store(scale_addr, tl.zeros((CHUNKS_PER_HEAD // 4,), dtype=tl.int32))
else:
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
scale_addrs = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ qb_indices * scale_stride_k
)
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
return
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
offsets = tl.arange(0, HEAD_DIM)
x = tl.load(input_base + offsets).to(tl.float32)
rope_abs_start: tl.constexpr = NOPE_DIM
pos = tl.load(positions_ptr + pid_token)
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
is_rope = offsets >= rope_abs_start
rope_local = offsets - rope_abs_start
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
tl.float32
)
cs_idx = tl.maximum(rope_local >> 1, 0)
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
x_add = x * cos_v + x_partner * sin_v
x_sub = x * cos_v - x_partner * sin_v
is_even = (rope_local & 1) == 0
rotated = tl.where(is_even, x_add, x_sub)
x = tl.where(is_rope, rotated, x)
if not QUANTIZE:
out_base = (
out_ptr
+ g * out_stride_group
+ pid_token * out_stride_token
+ qb_start * QUANT_GROUP_SIZE
)
tl.store(out_base + offsets, x)
return
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
scale_raw = block_absmax * (1.0 / fp8_max)
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
scales_exp = tl.reshape(
tl.broadcast_to(
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
),
(HEAD_DIM,),
)
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
out_base = (
out_ptr
+ g * out_stride_group
+ pid_token * out_stride_token
+ qb_start * QUANT_GROUP_SIZE
)
tl.store(out_base + offsets, x_quant)
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
if TMA_ALIGNED_SCALES:
scale_bits = scales.to(tl.int32, bitcast=True)
ue8m0_bytes = (scale_bits >> 23) & 0xFF
packed_val = tl.sum(
tl.reshape(ue8m0_bytes, (CHUNKS_PER_HEAD // 4, 4))
<< (tl.arange(0, 4)[None, :] * 8),
axis=1,
)
packed_offsets = tl.arange(0, CHUNKS_PER_HEAD // 4)
scale_addr = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ (head_in_group * (CHUNKS_PER_HEAD // 4) + packed_offsets)
* scale_stride_k
)
tl.store(scale_addr, packed_val)
else:
scale_addrs = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ qb_indices * scale_stride_k
)
tl.store(scale_addrs, scales)
def dispatch( # type: ignore[override]
self,
*,
heads_per_group: int,
head_dim: int,
nope_dim: int,
rope_dim: int,
quant_group_size: int,
runtime_fp8_max: float | None = None,
runtime_chunks_per_head: int | None = None,
runtime_nope_dim: int | None = None,
runtime_half_rope: int | None = None,
**compile_key_fields: bool,
) -> CompileKey:
fp8_max = (
runtime_fp8_max
if runtime_fp8_max is not None
else torch.finfo(torch.float8_e4m3fn).max
)
chunks_per_head = (
runtime_chunks_per_head
if runtime_chunks_per_head is not None
else head_dim // quant_group_size
)
nope_dim = runtime_nope_dim if runtime_nope_dim is not None else nope_dim
half_rope = (
runtime_half_rope if runtime_half_rope is not None else rope_dim // 2
)
return self.CompileKey(
**compile_key_fields,
heads_per_group=heads_per_group,
fp8_max=fp8_max,
quant_group_size=quant_group_size,
chunks_per_head=chunks_per_head,
nope_dim=nope_dim,
half_rope=half_rope,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if hf_config is None:
return []
tp_size = vllm_config.parallel_config.tensor_parallel_size
num_heads = int(getattr(hf_config, "num_attention_heads", 0) or 0)
num_groups = int(getattr(hf_config, "o_groups", 0) or 0)
if num_heads <= 0 or num_groups <= 0:
return []
local_heads = num_heads // tp_size
local_groups = num_groups // tp_size
if local_groups <= 0:
return []
capability = current_platform.get_device_capability()
if capability is None:
return []
head_dim = int(getattr(hf_config, "head_dim", 512) or 512)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 64) or 64)
return self._trace_dispatch(self.dispatch)(
heads_per_group=local_heads // local_groups,
head_dim=head_dim,
nope_dim=head_dim - rope_dim,
rope_dim=rope_dim,
quant_group_size=128,
tma_aligned_scales=capability.major >= 10,
launch_pdl=current_platform.is_arch_support_pdl(),
quantize=True,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
head_dim = compile_key.chunks_per_head * compile_key.quant_group_size
out_dim = compile_key.heads_per_group * head_dim
scale_dtype = torch.int32 if compile_key.tma_aligned_scales else torch.float32
return dict(
o=TritonWarmupTensor(
torch.bfloat16,
shape=(1, compile_key.heads_per_group, head_dim),
),
positions=TritonWarmupTensor(torch.int64),
cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.half_rope * 2),
),
out_buf=TritonWarmupTensor(
torch.float8_e4m3fn if compile_key.quantize else torch.bfloat16,
shape=(compile_key.heads_per_group, 1, out_dim),
strides=(out_dim, out_dim, 1),
),
scale_buf=TritonWarmupTensor(
scale_dtype,
shape=(compile_key.heads_per_group, 1, 16),
strides=(16, 16, 16),
),
num_tokens=1,
heads_per_group=compile_key.heads_per_group,
quant_group_size=compile_key.quant_group_size,
chunks_per_head=compile_key.chunks_per_head,
nope_dim=compile_key.nope_dim,
half_rope=compile_key.half_rope,
tma_aligned_scales=compile_key.tma_aligned_scales,
quantize=compile_key.quantize,
fp8_max=compile_key.fp8_max,
launch_pdl=compile_key.launch_pdl,
grid=(1, compile_key.heads_per_group),
)
@kernel_launcher
def __call__(
self,
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
out_buf: torch.Tensor,
scale_buf: torch.Tensor,
num_tokens: int,
*,
heads_per_group: int,
quant_group_size: int,
chunks_per_head: int,
nope_dim: int,
half_rope: int,
tma_aligned_scales: bool,
quantize: bool,
fp8_max: float,
launch_pdl: bool,
grid: tuple[int, int],
) -> LaunchSpec:
return grid, dict(
out_ptr=out_buf,
scale_ptr=scale_buf,
o_stride_token=o.stride(0),
o_stride_head=o.stride(1),
cache_stride_pos=cos_sin_cache.stride(0),
out_stride_group=out_buf.stride(0),
out_stride_token=out_buf.stride(1),
scale_stride_group=scale_buf.stride(0) if quantize else 0,
scale_stride_k=scale_buf.stride(2) if quantize else 0,
eps=1e-10,
QUANT_GROUP_SIZE=quant_group_size,
CHUNKS_PER_HEAD=chunks_per_head,
NOPE_DIM=nope_dim,
HALF_ROPE=half_rope,
QUANTIZE=quantize,
TMA_ALIGNED_SCALES=tma_aligned_scales,
launch_pdl=launch_pdl,
num_stages=1,
num_warps=1,
)
def fused_inv_rope_fp8_quant(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
n_groups: int,
heads_per_group: int,
nope_dim: int = 448,
rope_dim: int = 64,
quant_group_size: int = 128,
tma_aligned_scales: bool = False,
quantize: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused inverse RoPE + block-scaled FP8 quantization.
Args:
o: Attention output [num_tokens, num_heads, head_dim] bf16.
positions: Token positions [num_tokens] int64.
cos_sin_cache: Precomputed [max_pos, rope_dim] with cos||sin.
n_groups: Number of output groups.
heads_per_group: Heads per group.
nope_dim: Non-RoPE dimensions per head (default 448).
rope_dim: RoPE dimensions per head (default 64).
quant_group_size: FP8 quantization block size (default 128).
tma_aligned_scales: Output INT32 packed UE8M0 for SM100 (True)
or FP32 for SM90 (False).
quantize: Quantize the rotated output to FP8 and return its scales.
Returns:
Rotated output in [T, G, D] and its FP8 scales. The scale tensor is
empty when quantization is disabled.
"""
from vllm.utils.deep_gemm import get_tma_aligned_size
num_tokens, num_heads, head_dim = o.shape
assert num_heads == n_groups * heads_per_group
assert head_dim == nope_dim + rope_dim
assert head_dim % quant_group_size == 0
assert rope_dim % 2 == 0
assert cos_sin_cache.shape[-1] == rope_dim
assert cos_sin_cache.dtype == torch.float32
d = heads_per_group * head_dim
num_scale_blocks = d // quant_group_size
chunks_per_head = head_dim // quant_group_size
fp8_dtype = torch.float8_e4m3fn
fp8_max = torch.finfo(fp8_dtype).max
tma_aligned_T = get_tma_aligned_size(num_tokens, 4) if quantize else num_tokens
if quantize and tma_aligned_scales:
assert chunks_per_head % 4 == 0
packed_sf_k = (num_scale_blocks + 3) // 4
scale_inner = packed_sf_k
elif quantize:
scale_inner = num_scale_blocks
else:
scale_inner = 0
# Run kernel through a custom op so inductor sees an opaque boundary.
# It's a pytorch bug, see https://github.com/vllm-project/vllm/issues/41106
out_buf, scale_buf = torch.ops.vllm.fused_inv_rope_fp8_quant_kernel(
o,
positions,
cos_sin_cache,
heads_per_group,
quant_group_size,
chunks_per_head,
nope_dim,
rope_dim // 2,
tma_aligned_scales,
fp8_max,
tma_aligned_T,
num_tokens,
n_groups,
d,
scale_inner,
quantize,
)
output = out_buf.transpose(0, 1)
scales = scale_buf.transpose(0, 1) if quantize else scale_buf
return output, scales
def _fused_inv_rope_fp8_quant_kernel_impl(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
heads_per_group: int,
quant_group_size: int,
chunks_per_head: int,
nope_dim: int,
half_rope: int,
tma_aligned_scales: bool,
fp8_max: float,
tma_aligned_T: int,
num_tokens: int,
n_groups: int,
d: int,
scale_inner: int,
quantize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
scale_dtype = torch.int32 if tma_aligned_scales else torch.float32
out_buf = torch.empty(
(n_groups, num_tokens, d),
dtype=torch.float8_e4m3fn if quantize else o.dtype,
device=o.device,
)
if quantize:
scale_buf = torch.empty(
n_groups * scale_inner * tma_aligned_T,
dtype=scale_dtype,
device=o.device,
).as_strided(
(n_groups, num_tokens, scale_inner),
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
)
else:
scale_buf = torch.empty(0, dtype=scale_dtype, device=o.device)
grid = (tma_aligned_T, n_groups * heads_per_group)
launch_pdl = current_platform.is_arch_support_pdl()
_FUSED_INV_ROPE_FP8_QUANT_KERNEL(
o,
positions,
cos_sin_cache,
out_buf,
scale_buf,
num_tokens,
heads_per_group=heads_per_group,
quant_group_size=quant_group_size,
chunks_per_head=chunks_per_head,
nope_dim=nope_dim,
half_rope=half_rope,
tma_aligned_scales=tma_aligned_scales,
quantize=quantize,
fp8_max=fp8_max,
launch_pdl=launch_pdl,
grid=grid,
)
return out_buf, scale_buf
def _fused_inv_rope_fp8_quant_kernel_fake(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
heads_per_group: int,
quant_group_size: int,
chunks_per_head: int,
nope_dim: int,
half_rope: int,
tma_aligned_scales: bool,
fp8_max: float,
tma_aligned_T: int,
num_tokens: int,
n_groups: int,
d: int,
scale_inner: int,
quantize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
scale_dtype = torch.int32 if tma_aligned_scales else torch.float32
out_buf = torch.empty(
(n_groups, num_tokens, d),
dtype=torch.float8_e4m3fn if quantize else o.dtype,
device=o.device,
)
if not quantize:
return out_buf, torch.empty(0, dtype=scale_dtype, device=o.device)
scale_buf = torch.empty(
n_groups * scale_inner * tma_aligned_T,
dtype=scale_dtype,
device=o.device,
).as_strided(
(n_groups, num_tokens, scale_inner),
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
)
return out_buf, scale_buf
direct_register_custom_op(
op_name="fused_inv_rope_fp8_quant_kernel",
op_func=_fused_inv_rope_fp8_quant_kernel_impl,
fake_impl=_fused_inv_rope_fp8_quant_kernel_fake,
)
_FUSED_INV_ROPE_FP8_QUANT_KERNEL = FusedInvRopeFP8QuantKernel()
@@ -1,340 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused MTP-input RMSNorm: enorm (with mask-zero at position 0) + hnorm.
Replaces the eager sequence at the top of the MTP draft forward:
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
inputs_embeds = self.enorm(inputs_embeds)
previous_hidden_states = previous_hidden_states.view(-1, hc_mult, H)
previous_hidden_states = self.hnorm(previous_hidden_states)
which lowers to ~6 small kernels (CompareEq, where, Fill, enorm rms_norm,
hnorm rms_norm, plus aten elementwise helpers) on the breakable-cudagraph
path. Math is preserved: positions==0 → masked row → zero RMS output
regardless of weight.
A single grid (T, hc_mult+1) drives both norms: task 0 is enorm on
inputs_embeds[token, :], task k+1 is hnorm on previous_hidden_states[token, k, :].
"""
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
@triton.jit
def _rmsnorm_row(
x,
w_ptr,
out_row_ptr,
block,
mask,
eps,
HIDDEN: tl.constexpr,
):
x = x.to(tl.float32)
variance = tl.sum(x * x, axis=0) / HIDDEN
rrms = tl.rsqrt(variance + eps)
w = tl.load(w_ptr + block, mask=mask, other=0.0).to(tl.float32)
y = x * rrms * w
tl.store(out_row_ptr + block, y.to(out_row_ptr.dtype.element_ty), mask=mask)
class FusedMTPInputRMSNormKernel(
VllmTritonJitKernel["FusedMTPInputRMSNormKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
hidden: int
hc_mult: int
block_size: int
eps: float
@staticmethod
@triton.jit
def kernel(
inputs_embeds_ptr,
positions_ptr,
prev_hidden_ptr,
enorm_weight_ptr,
hnorm_weight_ptr,
enorm_out_ptr,
hnorm_out_ptr,
eps,
HIDDEN: tl.constexpr,
HC_MULT: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
# int64 token index so per-token offsets don't overflow int32 at
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
token_idx = tl.program_id(0).to(tl.int64)
pid_task = tl.program_id(1)
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
if pid_task == 0:
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
pos = tl.load(positions_ptr + token_idx)
keep = pos != 0
x = tl.load(
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
)
x = tl.where(keep, x, 0.0)
_rmsnorm_row(
x,
enorm_weight_ptr,
enorm_out_ptr + token_idx * HIDDEN,
block,
mask,
eps,
HIDDEN,
)
else:
# hnorm path: load prev_hidden[token, slot, :].
slot = pid_task - 1
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
hnorm_weight_ptr,
hnorm_out_ptr + row_offset,
block,
mask,
eps,
HIDDEN,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
hidden: int,
hc_mult: int,
eps: float,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
hidden=hidden,
hc_mult=hc_mult,
block_size=next_power_of_2(hidden),
eps=eps,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
hidden=int(hf_config.hidden_size),
hc_mult=int(hf_config.hc_mult),
eps=float(hf_config.rms_norm_eps),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
inputs_embeds=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
positions=TritonWarmupTensor(torch.int64),
previous_hidden_states=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hc_mult, compile_key.hidden),
),
enorm_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
hnorm_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
_outputs=(
TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hc_mult, compile_key.hidden),
),
),
eps=compile_key.eps,
hc_mult=compile_key.hc_mult,
)
@kernel_launcher
def __call__(
self,
inputs_embeds: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
enorm_weight: torch.Tensor,
hnorm_weight: torch.Tensor,
eps: float,
hc_mult: int,
*,
_outputs: tuple[Any, Any] | None = None,
) -> LaunchSpec:
num_tokens, hidden = inputs_embeds.shape
if _outputs is None:
assert inputs_embeds.ndim == 2
assert previous_hidden_states.ndim == 3
assert previous_hidden_states.shape[1] == hc_mult
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
"token dim mismatch"
)
assert (
hidden
== previous_hidden_states.shape[2]
== enorm_weight.shape[0]
== hnorm_weight.shape[0]
)
assert (
inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
)
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
_outputs = (
torch.empty_like(inputs_embeds),
torch.empty_like(previous_hidden_states),
)
enorm_out, hnorm_out = _outputs
grid = None if num_tokens == 0 else (num_tokens, hc_mult + 1)
return (
grid,
dict(
prev_hidden_ptr=previous_hidden_states,
enorm_out_ptr=enorm_out,
hnorm_out_ptr=hnorm_out,
HIDDEN=hidden,
HC_MULT=hc_mult,
BLOCK_SIZE=next_power_of_2(hidden),
),
_outputs,
)
class MTPSharedHeadRMSNormKernel(
VllmTritonJitKernel["MTPSharedHeadRMSNormKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
hidden: int
block_size: int
eps: float
@staticmethod
@triton.jit
def kernel(
x_ptr,
weight_ptr,
out_ptr,
eps,
HIDDEN: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
weight_ptr,
out_ptr + token_idx * HIDDEN,
block,
mask,
eps,
HIDDEN,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
hidden: int,
eps: float,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
hidden=hidden,
block_size=next_power_of_2(hidden),
eps=eps,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
hidden=int(hf_config.hidden_size),
eps=float(hf_config.rms_norm_eps),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
hidden_states=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
_output=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
eps=compile_key.eps,
)
@kernel_launcher
def __call__(
self,
hidden_states: torch.Tensor,
weight: torch.Tensor,
eps: float,
*,
_output: Any | None = None,
) -> LaunchSpec:
num_tokens, hidden = hidden_states.shape
if _output is None:
assert hidden_states.ndim == 2
assert hidden_states.is_contiguous()
assert weight.is_contiguous()
_output = torch.empty_like(hidden_states)
grid = None if num_tokens == 0 else (num_tokens,)
return (
grid,
dict(
x_ptr=hidden_states,
out_ptr=_output,
HIDDEN=hidden,
BLOCK_SIZE=next_power_of_2(hidden),
),
_output,
)
_FUSED_MTP_INPUT_RMSNORM_KERNEL = FusedMTPInputRMSNormKernel()
_MTP_SHARED_HEAD_RMSNORM_KERNEL = MTPSharedHeadRMSNormKernel()
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
class SavePartialStatesKernel(
VllmTritonJitKernel["SavePartialStatesKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
head_size: int
triton_block_size: int
state_width: int
compress_ratio: int
kv_stride: int
score_stride: int
ape_stride: int
state_cache_stride0: int
state_cache_stride1: int
block_size: int
@staticmethod
@triton.jit
def kernel(
kv_ptr,
kv_stride,
score_ptr,
score_stride,
ape_ptr,
ape_stride,
positions_ptr,
state_cache_ptr,
state_cache_stride0,
state_cache_stride1,
slot_mapping_ptr,
block_size,
HEAD_SIZE: tl.constexpr,
TRITON_BLOCK_SIZE: tl.constexpr,
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
STATE_WIDTH: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
):
token_idx = tl.program_id(0)
slot_id = tl.load(slot_mapping_ptr + token_idx)
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
# by vLLM). During CUDA graph replay the batch may contain padding
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
# illegal memory access.
if slot_id < 0:
return
block_idx = slot_id // block_size
pos_in_block = slot_id % block_size
base_ptr = (
state_cache_ptr
+ block_idx * state_cache_stride0
+ pos_in_block * state_cache_stride1
)
block = tl.arange(0, TRITON_BLOCK_SIZE)
mask = block < HEAD_SIZE
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
tl.store(base_ptr + block, kv, mask=mask)
# Fused: score += ape[position % compress_ratio]
position = tl.load(positions_ptr + token_idx)
ape_row = position % COMPRESS_RATIO
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
tl.store(
base_ptr + STATE_WIDTH + block,
score + ape,
mask=mask,
)
def dispatch( # type: ignore[override]
self,
*,
head_size: int,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
head_size=head_size,
triton_block_size=next_power_of_2(head_size),
)
def get_warmup_keys(
self,
*,
head_dim: int,
compress_ratio: int,
) -> list[CompileKey]:
if head_dim <= 0 or compress_ratio not in (4, 128):
return []
coefficient = 2 if compress_ratio == 4 else 1
return self._trace_dispatch(self.dispatch)(
head_size=coefficient * head_dim,
state_width=coefficient * head_dim,
compress_ratio=compress_ratio,
kv_stride=2 * coefficient * head_dim,
score_stride=2 * coefficient * head_dim,
ape_stride=coefficient * head_dim,
state_cache_stride0=8 * coefficient * head_dim,
state_cache_stride1=2 * coefficient * head_dim,
block_size=4 if compress_ratio == 4 else 8,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
kv=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.head_size),
strides=(compile_key.kv_stride, 1),
),
score=TritonWarmupTensor(
torch.float32,
shape=(1, 1),
strides=(compile_key.score_stride, 1),
),
ape=TritonWarmupTensor(
torch.float32,
shape=(1, 1),
strides=(compile_key.ape_stride, 1),
),
positions=TritonWarmupTensor(torch.int64),
state_cache=TritonWarmupTensor(
torch.float32,
shape=(1, 1, compile_key.state_width),
strides=(
compile_key.state_cache_stride0,
compile_key.state_cache_stride1,
1,
),
),
slot_mapping=TritonWarmupTensor(torch.int64),
block_size=compile_key.block_size,
state_width=compile_key.state_width,
compress_ratio=compile_key.compress_ratio,
)
@kernel_launcher
def __call__(
self,
kv: torch.Tensor,
score: torch.Tensor,
ape: torch.Tensor,
positions: torch.Tensor,
state_cache: torch.Tensor,
slot_mapping: torch.Tensor,
block_size: int,
state_width: int,
compress_ratio: int,
pdl_kwargs: dict | None = None,
) -> LaunchSpec:
"""Write packed [kv, score+ape] partial states into the compressor cache.
One program per token; pads (slot_id == -1) are skipped.
"""
num_actual = slot_mapping.shape[0]
head_size = kv.shape[-1]
return (num_actual,), dict(
kv_stride=kv.stride(0),
score_stride=score.stride(0),
ape_stride=ape.stride(0),
state_cache_stride0=state_cache.stride(0),
state_cache_stride1=state_cache.stride(1),
block_size=block_size,
HEAD_SIZE=head_size,
TRITON_BLOCK_SIZE=next_power_of_2(head_size),
STATE_WIDTH=state_width,
COMPRESS_RATIO=compress_ratio,
**(pdl_kwargs or {}),
)
_SAVE_PARTIAL_STATES_KERNEL = SavePartialStatesKernel()
-52
View File
@@ -1,52 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepseekV4 rotary embedding initialization."""
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.base import RotaryEmbedding
def build_deepseek_v4_rope(
config,
*,
head_dim: int,
rope_head_dim: int,
max_position_embeddings: int,
compress_ratio: int,
) -> RotaryEmbedding:
rope_parameters = config.rope_parameters
# Newer checkpoints nest per-layer-type rope dicts ({"main", "compress"});
# older ones ship a single flat dict shared by all layer types.
if isinstance(rope_parameters.get("main"), dict) and isinstance(
rope_parameters.get("compress"), dict
):
key = "compress" if compress_ratio > 1 else "main"
rope_parameters = dict(rope_parameters[key])
else:
rope_parameters = dict(rope_parameters)
rope_parameters["rope_theta"] = (
config.compress_rope_theta if compress_ratio > 1 else config.rope_theta
)
if compress_ratio > 1 and rope_parameters["rope_type"] != "default":
# YaRN applies only to compressor (CSA/HCA) layers.
rope_parameters["rope_type"] = (
"deepseek_llama_scaling"
if rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
else:
# Sliding-window layers use plain RoPE (theta=rope_theta, no YaRN).
rope_parameters["rope_type"] = "deepseek_yarn"
rope_parameters["factor"] = 1.0
rope_parameters["original_max_position_embeddings"] = max_position_embeddings
rope_parameters["mscale"] = 0 # Disable mscale
rope_parameters["mscale_all_dim"] = 0 # Disable mscale
rope_parameters["is_deepseek_v4"] = True
rope_parameters["rope_dim"] = rope_head_dim
return get_rope(
head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
is_neox_style=False,
)
-476
View File
@@ -1,476 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 vision tower (ViT + aligner) with TP-sharded linears.
Ported from the official reference implementation
(deepseek-ai/DeepSeek-V4-Flash-Vision-Exp). Weight names match the HF
checkpoint so no renaming is needed at load time. Attention and MLP weights
are tensor-parallel sharded (replicated when the vision head count is not
divisible by TP size, or under ``--mm-encoder-tp-mode data``); the patch
embed and norms are replicated, so the residual stream is full-width on
every rank.
"""
import itertools
from functools import lru_cache
import torch
import torch.nn.functional as F
from torch import nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.models.vision import (
get_load_balance_assignment,
is_vit_use_data_parallel,
)
from vllm.utils.torch_utils import async_tensor_h2d
def _compute_vision_cos_sin(
n_h: int, n_w: int, dim: int, theta: float
) -> tuple[torch.Tensor, torch.Tensor]:
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
hpos = torch.arange(n_h).unsqueeze(1).expand(n_h, n_w)
wpos = torch.arange(n_w).unsqueeze(0).expand(n_h, n_w)
freqs = torch.stack([hpos, wpos], dim=-1).reshape(-1, 2, 1).float()
freqs = (freqs * inv_freq).flatten(1)
return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)
@lru_cache(8)
def get_vision_cos_sin(
n_h: int, n_w: int, dim: int, theta: float
) -> tuple[torch.Tensor, torch.Tensor]:
return _compute_vision_cos_sin(n_h, n_w, dim, theta)
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x1, x2 = x.float().chunk(2, dim=-1)
return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(dtype)
class DeepseekV4RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
def forward(self, x: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x = x.float()
x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)
return (self.weight * x).to(dtype)
class DeepseekV4PatchEmbed(nn.Module):
def __init__(self, config):
super().__init__()
# Replicated: the residual stream is full-width on every rank.
self.proj = ReplicatedLinear(
3 * config.vision_patch_size**2,
config.vision_dim,
bias=True,
quant_config=None,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out, _ = self.proj(x.flatten(1))
return out
class DeepseekV4VisionAttention(nn.Module):
def __init__(self, config, prefix: str = ""):
super().__init__()
# Convention from minimax_m3/siglip: compute the TP/DP choice locally
# instead of threading a flag through every constructor.
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
self.tp_size = (
1 if use_data_parallel else get_tensor_model_parallel_world_size()
)
self.n_heads = config.vision_n_heads // self.tp_size
self.head_dim = config.vision_dim // config.vision_n_heads
self.hidden_size = config.vision_dim
self.wqkv = QKVParallelLinear(
config.vision_dim,
self.head_dim,
config.vision_n_heads,
bias=True,
quant_config=None,
prefix=f"{prefix}.wqkv",
disable_tp=use_data_parallel,
)
self.wo = RowParallelLinear(
config.vision_dim,
config.vision_dim,
bias=True,
quant_config=None,
prefix=f"{prefix}.wo",
disable_tp=use_data_parallel,
)
self.attn = MMEncoderAttention(
num_heads=self.n_heads,
head_size=self.head_dim,
prefix=f"{prefix}.attn",
)
def forward(
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
n = x.size(0)
qkv, _ = self.wqkv(x)
q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in qkv.chunk(3, -1))
q = apply_rotary(q, cos, sin).unsqueeze(0) # (b=1, n, h, d)
k = apply_rotary(k, cos, sin).unsqueeze(0)
# Dense (cu_seqlens=None): one image per call. Varlen (cu_seqlens
# set): packed multi-image batch for encoder CUDA graph replay.
o = self.attn(
q, k, v.unsqueeze(0), cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
)
out, _ = self.wo(o.reshape(n, -1))
return out
class DeepseekV4VisionMLP(nn.Module):
def __init__(self, config, prefix: str = ""):
super().__init__()
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
self.w1 = MergedColumnParallelLinear(
config.vision_dim,
[config.vision_inter_dim] * 2,
bias=False,
quant_config=None,
prefix=f"{prefix}.w1",
disable_tp=use_data_parallel,
)
self.w2 = RowParallelLinear(
config.vision_inter_dim,
config.vision_dim,
bias=False,
quant_config=None,
prefix=f"{prefix}.w2",
disable_tp=use_data_parallel,
)
self.act_fn = SiluAndMul()
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.w1(x)
out, _ = self.w2(self.act_fn(gate_up))
return out
class DeepseekV4VisionBlock(nn.Module):
def __init__(self, config, prefix: str = ""):
super().__init__()
self.norm1 = DeepseekV4RMSNorm(config.vision_dim)
self.attn = DeepseekV4VisionAttention(config, prefix=f"{prefix}.attn")
self.norm2 = DeepseekV4RMSNorm(config.vision_dim)
self.mlp = DeepseekV4VisionMLP(config, prefix=f"{prefix}.mlp")
def forward(
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
x = x + self.attn(self.norm1(x), cos, sin, cu_seqlens, max_seqlen)
return x + self.mlp(self.norm2(x))
class DeepseekV4ViT(nn.Module):
"""DeepSeek-V4 ViT: full bidirectional attention per image, 2D RoPE."""
def __init__(self, config):
super().__init__()
self.rope_dim = config.vision_dim // config.vision_n_heads // 2
self.rope_theta = config.vision_rope_theta
self.patch_embed = DeepseekV4PatchEmbed(config)
self.blocks = nn.ModuleList(
[
DeepseekV4VisionBlock(config, prefix=f"blocks.{i}")
for i in range(config.vision_n_layers)
]
)
self.norm = DeepseekV4RMSNorm(config.vision_dim)
def forward(
self, patches: torch.Tensor, n_vit_h: int, n_vit_w: int
) -> torch.Tensor:
x = self.patch_embed(patches)
cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta)
cos = async_tensor_h2d(cos, x.device)
sin = async_tensor_h2d(sin, x.device)
for block in self.blocks:
x = block(x, cos, sin)
return self.norm(x)
def forward_packed(
self,
patches: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
) -> torch.Tensor:
"""Varlen path for encoder CUDA graphs: multiple images packed along
rows, per-image attention via ``cu_seqlens``, RoPE tables precomputed
on device. Pure-tensor (no host reads, no H2D), safe to capture."""
x = self.patch_embed(patches)
for block in self.blocks:
x = block(x, cos, sin, cu_seqlens, max_seqlen)
return self.norm(x)
def build_packed_vit_metadata(
grids: list[list[int]] | list[tuple[int, int]],
*,
rope_dim: int,
rope_theta: float,
device: torch.device,
max_seqlen_override: int | None = None,
cached: bool = True,
) -> dict[str, torch.Tensor]:
"""Precompute packed-batch ViT metadata for ``DeepseekV4ViT.forward_packed``.
Args:
grids: ``[n_vit_h, n_vit_w]`` per image, in packing order.
rope_dim: RoPE embedding dimension of the ViT.
rope_theta: RoPE base frequency of the ViT.
device: Device for the returned tensors (except ``max_seqlen``,
which stays on the host).
max_seqlen_override: Worst-case value baked in at CUDA graph capture
(the attention wrapper reads ``max_seqlen`` on the host, so the
capture-time value becomes a graph constant).
cached: Use the shared ``lru_cache`` for RoPE tables. Capture-time
dummy grids pass ``False`` to avoid evicting real entries.
"""
cos_sin = get_vision_cos_sin if cached else _compute_vision_cos_sin
cos_list: list[torch.Tensor] = []
sin_list: list[torch.Tensor] = []
cu_seqlens = [0]
max_seqlen = 0
for n_h, n_w in grids:
cos, sin = cos_sin(n_h, n_w, rope_dim, rope_theta)
cos_list.append(cos)
sin_list.append(sin)
n = n_h * n_w
cu_seqlens.append(cu_seqlens[-1] + n)
max_seqlen = max(max_seqlen, n)
if cos_list:
cos = async_tensor_h2d(torch.cat(cos_list), device)
sin = async_tensor_h2d(torch.cat(sin_list), device)
else:
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
if max_seqlen_override is not None:
max_seqlen = max_seqlen_override
return {
"vit_cos": cos,
"vit_sin": sin,
"cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32),
# Read on the host by the attention wrapper; keep on CPU.
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
}
def build_packed_merge_metadata(
grids: list[list[int]] | list[tuple[int, int]],
downsample_ratio: int,
*,
device: torch.device,
dtype: torch.dtype,
) -> dict[str, torch.Tensor]:
"""Gather indices/mask for ``DeepseekV4Aligner.forward_packed``.
For each aligner output row, the ``r x r`` source patch rows in the
packed ViT output; positions past the image edge (the eager path pads
them with zeros) get mask 0 and a clamped in-range index.
"""
r = downsample_ratio
idx_list: list[torch.Tensor] = []
mask_list: list[torch.Tensor] = []
offset = 0
for n_h, n_w in grids:
rows = -(-n_h // r)
cols = -(-n_w // r)
bi = torch.arange(rows).repeat_interleave(cols)
bj = torch.arange(cols).repeat(rows)
di = torch.arange(r).repeat_interleave(r)
dj = torch.arange(r).repeat(r)
hi = bi[:, None] * r + di[None, :]
wj = bj[:, None] * r + dj[None, :]
valid = (hi < n_h) & (wj < n_w)
idx = offset + hi.clamp(max=n_h - 1) * n_w + wj.clamp(max=n_w - 1)
idx_list.append(idx)
mask_list.append(valid)
offset += n_h * n_w
if idx_list:
merge_idx = async_tensor_h2d(torch.cat(idx_list), device)
merge_mask = async_tensor_h2d(
torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype
)
else:
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
return {"merge_idx": merge_idx, "merge_mask": merge_mask}
class DeepseekV4Aligner(nn.Module):
"""Spatial merge (downsample_ratio x downsample_ratio) + MLP projector."""
def __init__(self, config):
super().__init__()
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
self.downsample_ratio = config.vision_downsample_ratio
self.out_dim = config.hidden_size
in_dim = config.vision_dim * self.downsample_ratio**2
self.w1 = ColumnParallelLinear(
in_dim,
config.hidden_size,
bias=True,
quant_config=None,
disable_tp=use_data_parallel,
)
self.w2 = RowParallelLinear(
config.hidden_size,
config.hidden_size,
bias=True,
quant_config=None,
disable_tp=use_data_parallel,
)
def forward(self, x: torch.Tensor, n_vit_h: int, n_vit_w: int) -> torch.Tensor:
r = self.downsample_ratio
x = x.view(n_vit_h, n_vit_w, -1).permute(2, 0, 1)
x = F.pad(x, (0, -n_vit_w % r, 0, -n_vit_h % r))
x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).transpose(0, 1)
hidden, _ = self.w1(x)
out, _ = self.w2(F.gelu(hidden))
return out
def forward_packed(
self,
x: torch.Tensor,
merge_idx: torch.Tensor,
merge_mask: torch.Tensor,
) -> torch.Tensor:
"""Packed multi-image path for encoder CUDA graphs.
``merge_idx``/``merge_mask`` come from ``build_packed_merge_metadata``;
gather+mask reproduces the eager ``F.pad`` + ``F.unfold`` merge
exactly (unfold rows are channel-major, so the gathered patch rows
are transposed before flattening).
"""
r2 = self.downsample_ratio**2
m = merge_idx.shape[0]
gathered = (x[merge_idx] * merge_mask).view(m, r2, -1)
gathered = gathered.permute(0, 2, 1).reshape(m, -1)
hidden, _ = self.w1(gathered)
out, _ = self.w2(F.gelu(hidden))
return out
def run_dp_sharded_vision_tower(
vision_model: DeepseekV4ViT,
aligner: DeepseekV4Aligner,
patches: torch.Tensor,
vit_grid: list[list[int]],
) -> list[torch.Tensor]:
"""Run the ViT + aligner with images sharded across TP ranks.
Every rank holds the full tower weights (``--mm-encoder-tp-mode data``)
and receives the full ``patches`` batch. Images are assigned to ranks by
patch count (greedy load balancing), each rank encodes only its share,
and per-image embeddings are exchanged with one padded all-gather and
returned in the original image order.
Args:
vision_model: The (weight-replicated) ViT tower.
aligner: The (weight-replicated) spatial-merge projector.
patches: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` patches of all images.
vit_grid: ``[n_vit_h, n_vit_w]`` per image.
Returns:
One ``(n_aligner_rows, hidden_size)`` embedding tensor per image.
"""
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
sizes = [h * w for h, w in vit_grid]
cum_patches = [0, *itertools.accumulate(sizes)]
image_to_tp_rank, gpu_sample_counts, _ = get_load_balance_assignment(sizes, tp_size)
cum_sample_counts = [0, *itertools.accumulate(gpu_sample_counts)]
# Rows the aligner emits per image: each grid dim is padded up to a
# multiple of the merge ratio before r x r patches fold into one row.
r = aligner.downsample_ratio
rows_per_image = [-(-h // r) * -(-w // r) for h, w in vit_grid]
def rank_image_idxs(g: int) -> list[int]:
return image_to_tp_rank[cum_sample_counts[g] : cum_sample_counts[g + 1]]
# All-gather needs one shape on every rank; pad each rank's packed
# output to the largest per-rank row count (computable locally).
max_rows = max(
sum(rows_per_image[i] for i in rank_image_idxs(g)) for g in range(tp_size)
)
local_embeds = [
aligner(
vision_model(
patches[cum_patches[i] : cum_patches[i + 1]],
vit_grid[i][0],
vit_grid[i][1],
),
vit_grid[i][0],
vit_grid[i][1],
)
for i in rank_image_idxs(tp_rank)
]
if local_embeds:
embeds_local = torch.cat(local_embeds, dim=0)
else:
embeds_local = patches.new_zeros((0, aligner.out_dim))
if embeds_local.shape[0] < max_rows:
embeds_local = torch.cat(
[
embeds_local,
embeds_local.new_zeros(
(max_rows - embeds_local.shape[0], embeds_local.shape[1])
),
],
dim=0,
)
gathered = tensor_model_parallel_all_gather(embeds_local.contiguous(), dim=0)
out: list[torch.Tensor] = [None] * len(vit_grid) # type: ignore[list-item]
for g in range(tp_size):
offset = g * max_rows
for i in rank_image_idxs(g):
out[i] = gathered[offset : offset + rows_per_image[i]]
offset += rows_per_image[i]
return out
-393
View File
@@ -1,393 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 vision variant (e.g. DeepSeek-V4-Flash-Vision-Exp).
Thin multimodal wrapper around the text-only ``DeepseekV4ForCausalLM``:
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
sentinel positions; four learned vectors (``image_start`` / ``image_pad`` /
``image_newline`` / ``image_end``) fill the remaining sentinel positions.
- Image placeholders (``<|deepseek_image|>``) are expanded by the processor
in ``common/mm_preprocess.py`` into sentinel blocks borrowing reserved
in-vocab tokens ``<|place_holder_mm_span_0431|>``..``_0435|>``
(see ``common/vision.py`` for the tower itself).
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
the model so the MoE router can apply ``bias_vl`` to image tokens
(``requires_raw_input_tokens``).
"""
from collections.abc import Iterable
import torch
from torch import nn
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsEagle3,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItem
from .mm_preprocess import (
COMPRESS_PAD_TO,
IMAGE_PLACEHOLDER,
IMAGE_SENTINEL_BASE_ID,
DeepseekV4VLDummyInputsBuilder,
DeepseekV4VLMultiModalProcessor,
DeepseekV4VLProcessingInfo,
image_sentinel_mask,
)
from .vision import DeepseekV4Aligner, DeepseekV4ViT
def _make_deepseek_v4_vl_weights_mapper(
text_mapper: WeightsMapper, image_enabled: bool
) -> WeightsMapper:
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
orig_to_new_prefix = {
src: None if dst is None else f"language_model.{dst}"
for src, dst in text_mapper.orig_to_new_prefix.items()
}
if not image_enabled:
orig_to_new_prefix.update({"vision.": None, "aligner.": None, "image_": None})
return WeightsMapper(
orig_to_new_renaming=text_mapper.orig_to_new_renaming,
orig_to_new_prefix=orig_to_new_prefix,
orig_to_new_regex=text_mapper.orig_to_new_regex,
orig_to_new_stacked=text_mapper.orig_to_new_stacked,
orig_to_new_suffix={
**text_mapper.orig_to_new_suffix,
"head.weight": "language_model.lm_head.weight",
},
orig_to_new_substr={
**text_mapper.orig_to_new_substr,
# Draft models load the checkpoint's MTP weights separately from
# the target model.
"mtp.": None,
},
)
def stream_language_model_first(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield ``language_model.*`` weights as they arrive, then the rest.
Keeps the language model's weights in one contiguous group without
materializing the stream. Deferred weights are cloned so they do not keep
the loader's per-shard storage alive until the end of the load.
"""
rest = []
for name, tensor in weights:
if name.startswith("language_model."):
yield name, tensor
else:
rest.append((name, tensor.clone()))
yield from rest
@MULTIMODAL_REGISTRY.register_processor(
DeepseekV4VLMultiModalProcessor,
info=DeepseekV4VLProcessingInfo,
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
)
class DeepseekV4ForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3, SupportsLoRA
):
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
delegates through ``language_model`` via the protocol defaults.
"""
packed_modules_mapping = {
"gate_up_proj": ["w1", "w3"],
"fused_wqa_wkv": ["wq_a", "wkv"],
"fused_wkv_wgate": ["wkv", "wgate"],
# for visual encoder
"wqkv": ["wqkv"],
"w1": ["w1"],
}
# The MoE router needs raw token ids to detect image sentinel tokens
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
requires_raw_input_tokens = True
supports_tower_connector_lora = True
# Image spans widen the sparse-SWA window in-kernel and the placeholder
# starts with a compressor-alignment pad
mm_prefix_clamp_sliding_window = True
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality == "image":
return IMAGE_PLACEHOLDER
raise ValueError(f"Unsupported modality: {modality!r}")
def __init__(self, *, vllm_config, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config = model_config.hf_config
self.config = config
self.multimodal_config = model_config.multimodal_config
assert self.multimodal_config is not None
image_enabled = (
config.vision_n_layers > 0
and self.multimodal_config.get_limit_per_prompt("image") > 0
)
with self._mark_tower_model(vllm_config, {"image"}):
self.vision: DeepseekV4ViT | None = None
self.aligner: DeepseekV4Aligner | None = None
self.image_start: nn.Parameter | None = None
self.image_end: nn.Parameter | None = None
self.image_newline: nn.Parameter | None = None
self.image_pad: nn.Parameter | None = None
if image_enabled:
self.vision = DeepseekV4ViT(config)
self.aligner = DeepseekV4Aligner(config)
for name in (
"image_start",
"image_end",
"image_newline",
"image_pad",
):
setattr(
self,
name,
nn.Parameter(
torch.empty(config.hidden_size, dtype=torch.float32)
),
)
self.vision.to(dtype=model_config.dtype)
self.aligner.to(dtype=model_config.dtype)
with self._mark_language_model(vllm_config):
# The arch convertor routes any config with a vision tower to
# this wrapper class; mark the copy handed to the text backbone
# so it resolves to DeepseekV4ForCausalLM instead of recursing
# (with_hf_config deepcopies the config, the marker survives).
config._dsv4_vl_inner = True # type: ignore[attr-defined]
try:
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config,
prefix=maybe_prefix(prefix, "language_model"),
architectures=["DeepseekV4ForCausalLM"],
)
finally:
del config._dsv4_vl_inner # type: ignore[attr-defined]
# The outer mapper (see load_weights) fully resolves HF names into
# this wrapper's namespace before AutoWeightsLoader strips the
# "language_model." prefix and delegates to the child's load_weights,
# so the child's own mapper must be a no-op. Its suffix rules are not
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
# re-fire "head.weight" -> "lm_head.weight").
text_mapper = self.language_model.hf_to_vllm_mapper
self.language_model.hf_to_vllm_mapper = WeightsMapper()
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
self.language_model.make_empty_intermediate_tensors
)
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
text_mapper, image_enabled
)
self._weights_finalized = False
def _parse_and_validate_image_input(self, **kwargs: object) -> dict | None:
patches = kwargs.pop("patches", None)
if patches is None:
return None
vit_grid = kwargs.pop("vit_grid", None)
llm_grid = kwargs.pop("llm_grid", None)
perm = kwargs.pop("perm", None)
assert vit_grid is not None and llm_grid is not None and perm is not None
return {
"patches": patches,
"vit_grid": vit_grid,
"llm_grid": llm_grid,
"perm": perm,
}
def _encode_image(
self,
patches: torch.Tensor,
n_vit_h: int,
n_vit_w: int,
perm: torch.Tensor,
) -> torch.Tensor:
assert self.vision is not None and self.aligner is not None
image_embeds = self.aligner(
self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w
)
# Reorder into the N-layout block order used in the prompt.
return image_embeds[perm.to(image_embeds.device)]
def _process_image_input(
self,
patches: torch.Tensor,
vit_grid: torch.Tensor,
llm_grid: torch.Tensor,
perm: torch.Tensor,
) -> tuple[torch.Tensor, ...]:
assert self.vision is not None and self.aligner is not None
patches = patches.to(self.aligner.w1.weight.dtype)
embeds: list[torch.Tensor] = []
vit_offset = 0
llm_offset = 0
for (n_vit_h, n_vit_w), (n_llm_h, n_llm_w) in zip(
vit_grid.tolist(), llm_grid.tolist(), strict=True
):
n_vit = n_vit_h * n_vit_w
n_llm = n_llm_h * n_llm_w
embeds.append(
self._encode_image(
patches[vit_offset : vit_offset + n_vit],
n_vit_h,
n_vit_w,
perm[llm_offset : llm_offset + n_llm],
)
)
vit_offset += n_vit
llm_offset += n_llm
return tuple(embeds)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None or self.vision is None:
return []
return self._process_image_input(
image_input["patches"],
image_input["vit_grid"],
image_input["llm_grid"],
image_input["perm"],
)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
# All ids are in-vocab here: image-block sentinels are borrowed
# reserved tokens (their embedding rows are always overwritten below).
inputs_embeds = self.language_model.embed_input_ids(input_ids)
if self.image_start is not None:
# Branch-free sentinel overwrite: safe inside compiled/captured
# regions (no data-dependent control flow).
sentinel_mask = image_sentinel_mask(input_ids)
if is_multimodal is not None:
# IMAGE positions get vision embeddings via the merge below.
sentinel_mask = sentinel_mask & ~is_multimodal.to(input_ids.device)
table = torch.stack(
[
self.image_start,
self.image_pad,
self.image_pad,
self.image_newline,
self.image_end,
]
).to(inputs_embeds.dtype)
idx = (input_ids - IMAGE_SENTINEL_BASE_ID).clamp(0, 4)
inputs_embeds = torch.where(
sentinel_mask.unsqueeze(-1), table[idx], inputs_embeds
)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
assert is_multimodal is not None
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors=None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
return self.language_model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits_local(hidden_states)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.language_model.get_expert_mapping()
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
return self.language_model.get_mtp_target_hidden_states()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
child_finalizes = getattr(
self.language_model, "finalizes_weights_during_load", True
)
mapped = self.hf_to_vllm_mapper.apply(weights)
if child_finalizes:
# A child which finalizes inside load_weights must see all of its
# weights in one contiguous delegation from AutoWeightsLoader.
mapped = stream_language_model_first(mapped)
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
self._weights_finalized = child_finalizes
return loaded_params
def process_weights_after_loading(self) -> None:
# Backbones such as the ROCm implementation require this to run only
# after the loader's generic per-layer quantization finalization.
if getattr(self, "_weights_finalized", False):
return
self.language_model.process_weights_after_loading()
self._weights_finalized = True
def get_mm_mapping(self) -> MultiModelKeys:
"""Get the module prefixes in the multimodal model."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="aligner",
tower_model="vision.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
if modality != "image":
raise ValueError(f"Unsupported modality: {modality!r}")
patches = mm_kwargs.get("patches") if mm_kwargs else None
if patches is not None and isinstance(patches.data, torch.Tensor):
tower_tokens = patches.data.shape[0]
else:
tower_tokens = num_mm_embeds * self.config.vision_downsample_ratio**2
return tower_tokens, num_mm_embeds
-499
View File
@@ -1,499 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any, ClassVar, cast
import torch
from torch import nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
compress_norm_rope_store_triton,
compress_norm_rope_store_two_stage_triton,
)
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
from vllm.models.deepseek_v4.common.ops.save_partial_states import (
_SAVE_PARTIAL_STATES_KERNEL,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import (
KVCacheSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
)
def _prefer_two_stage_compressor() -> bool:
# Platforms that favor the triton variant of two-stage compressor split.
# Currently only tested on ROCm
return current_platform.is_rocm()
def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:
seq_lens_cpu = metadata.seq_lens_cpu_upper_bound
if seq_lens_cpu is None:
return None
query_lens = metadata.query_start_loc_cpu[1:] - metadata.query_start_loc_cpu[:-1]
starts = seq_lens_cpu - query_lens
starts_list = starts.tolist()
query_start_loc = metadata.query_start_loc_cpu.tolist()
return any(
start % 128 + query_start_loc[i + 1] - query_start_loc[i] >= 128
for i, start in enumerate(starts_list)
)
class CompressorBackend(AttentionBackend):
def __init__(self):
super().__init__()
@staticmethod
def get_name() -> str:
return "CompressorBackend"
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [MultipleOf(1)]
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [512, 1024]
@staticmethod
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
return CompressorMetadataBuilder
@dataclass
class CompressorMetadata:
block_table: torch.Tensor
slot_mapping: torch.Tensor
block_size: int
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
num_decode_tokens: int | None = None
c128_boundary: bool | None = None
class CompressorMetadataBuilder(AttentionMetadataBuilder):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert isinstance(self.kv_cache_spec, SlidingWindowMLASpec | MLAAttentionSpec)
mla_spec = cast(SlidingWindowMLASpec | MLAAttentionSpec, self.kv_cache_spec)
self.block_size = mla_spec.block_size
self.token_to_req_indices = torch.zeros(
self.vllm_config.scheduler_config.max_num_batched_tokens,
dtype=torch.int32,
device=self.device,
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> CompressorMetadata:
token_to_req_indices = common_attn_metadata.token_to_req_indices(
self.token_to_req_indices
)
num_decode_tokens = None
if _prefer_two_stage_compressor():
_, _, num_decode_tokens, _ = split_decodes_and_prefills(
common_attn_metadata, decode_threshold=1
)
async_spec_decode = (
self.vllm_config.scheduler_config.async_scheduling
and self.vllm_config.speculative_config is not None
)
return CompressorMetadata(
block_table=common_attn_metadata.block_table_tensor.clamp_(min=0),
slot_mapping=common_attn_metadata.slot_mapping,
block_size=self.block_size,
token_to_req_indices=token_to_req_indices,
num_decode_tokens=num_decode_tokens,
c128_boundary=(
_get_c128_boundary(common_attn_metadata)
if self.block_size == 8 and not async_spec_decode
else None
),
)
class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
def __init__(
self,
state_dim: int,
dtype: torch.dtype,
compress_ratio: int,
prefix: str,
):
super().__init__()
self.state_dim = state_dim
self.dtype = dtype
self.prefix = prefix
self.kv_cache = torch.tensor([])
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
assert self.dtype == torch.float32
assert compress_ratio in [4, 128]
coff = 1 + (compress_ratio == 4)
self.sliding_window = coff * compress_ratio
# Block size is constrained by tensor sharing between compressor states
# and KV blocks. Since compressor states share the same physical tensor
# as KV blocks, they must use the same page size.
# The KV block shape [256//4, head_dim] = [64, 584] determines:
# - C4 compressor block shape [4, 2*512*2*4] -> block_size = 4
# - C128 compressor block shape [8, 512*2*4] -> block_size = 8
# TODO(yifan): make block size automatically determined and configurable.
if compress_ratio == 4:
self.block_size = 4
elif compress_ratio == 128:
self.block_size = 8
else:
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
# full-cache rows share state pages with contiguous KV pages, so padding
# would break page matching.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
return SlidingWindowMLASpec( # only has one vector instead of K + V
block_size=self.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.state_dim,
dtype=self.dtype,
sliding_window=self.sliding_window,
alignment=576 if uses_fp8_ds_mla_layout else 512,
)
def forward(self): ...
def get_attn_backend(self) -> type[AttentionBackend]:
return CompressorBackend
class DeepseekCompressor(nn.Module):
"""DeepSeek V4 KV/score compressor.
Owns the linear / norm / state-cache / ape state and the shared forward
prologue (kv/score split, save_partial_states launch). The
compress → norm → RoPE → store step is dispatched to a triton kernel
(``compress_norm_rope_store_triton``) by default, except for the NVIDIA
head_dim=512 path which uses the CuTeDSL compressor kernels for better
performance.
"""
def __init__(
self,
vllm_config: VllmConfig,
compress_ratio: int,
hidden_size: int,
head_dim: int,
rotate: bool = False,
prefix: str = "",
k_cache_prefix="",
use_fp4_cache: bool = False,
):
super().__init__()
self.compress_ratio = compress_ratio
self.hidden_size = hidden_size
self.head_dim = head_dim
self.rotate = rotate
self.prefix = prefix
self.k_cache_prefix = k_cache_prefix
self.use_fp4_cache = use_fp4_cache
config = vllm_config.model_config.hf_config
self.rope_head_dim = config.qk_rope_head_dim
self.nope_head_dim = self.head_dim - self.rope_head_dim
self.rms_norm_eps = config.rms_norm_eps
self.device = current_platform.device_type
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
self.max_model_len = vllm_config.model_config.max_model_len
self.overlap = compress_ratio == 4
self.coff = 1 + self.overlap
# The head=512 cr>=128 no-overlap deep gather uses the two-stage
# compressor, which needs an fp32 scratch [max_batched, 512] for
# the intermediate compressed_kv.
# Currently only tested on ROCm
self._use_two_stage_fused_compressor = (
_prefer_two_stage_compressor() and head_dim == 512 and not self.overlap
)
self.max_num_batched_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
self._compress_scratch: torch.Tensor | None = None
if self._use_two_stage_fused_compressor:
self._compress_scratch = torch.empty(
self.max_num_batched_tokens,
self.head_dim,
dtype=torch.float32,
device=self.device,
)
state_dtype = torch.float32
self.ape = nn.Parameter(
torch.empty(
(compress_ratio, self.coff * self.head_dim),
dtype=state_dtype,
device=self.device,
),
requires_grad=False,
)
self.fused_wkv_wgate = MergedColumnParallelLinear(
self.hidden_size,
[self.coff * self.head_dim, self.coff * self.head_dim],
bias=False,
return_bias=False,
quant_config=None,
disable_tp=True,
prefix=f"{prefix}.fused_wkv_wgate",
)
self.norm = RMSNorm(self.head_dim, self.rms_norm_eps)
self.state_cache = CompressorStateCache(
state_dim=2 * self.coff * self.head_dim, # kv_state + score_state
dtype=state_dtype,
compress_ratio=compress_ratio,
prefix=f"{prefix}.state_cache",
)
# Save reference to static_forward_context for forward-time KV cache lookup.
# get_current_vllm_config() is only available during __init__, not forward.
self._static_forward_context = (
vllm_config.compilation_config.static_forward_context
)
if self.head_dim == 512:
assert not use_fp4_cache, (
"MXFP4 cache is only supported for indexer (head=128)"
)
self._quant_block = 64
self._token_stride = self.nope_head_dim + self.rope_head_dim * 2
self._scale_dim = self.nope_head_dim // 64 + 1 # 7 real + 1 pad
elif self.head_dim == 128:
if use_fp4_cache:
self._quant_block = MXFP4_BLOCK_SIZE
self._token_stride = self.head_dim // 2
self._scale_dim = self.head_dim // MXFP4_BLOCK_SIZE
else:
self._quant_block = 128
self._token_stride = self.head_dim
self._scale_dim = 4 # single float32 scale
else:
raise ValueError(
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
)
if vllm_config.kernel_config.enable_jit_warmup:
_SAVE_PARTIAL_STATES_KERNEL.register_warmup(
head_dim=self.head_dim,
compress_ratio=self.compress_ratio,
)
if current_platform.is_cuda() and self.head_dim == 512:
from vllm.models.deepseek_v4.nvidia.ops.sparse_attn_compress_cutedsl import ( # noqa: E501
_SPARSE_ATTN_COMPRESS_C128_BLOCK8_KERNEL,
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_C4_KERNEL,
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_FULL_C4_KERNEL,
_SPARSE_ATTN_NORM_ROPE_STORE_FULL_KERNEL,
_SPARSE_ATTN_NORM_ROPE_STORE_KERNEL,
)
store_full_kv = vllm_config.cache_config.cache_dtype != "fp8_ds_mla"
if self.compress_ratio == 4:
(
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_FULL_C4_KERNEL
if store_full_kv
else _SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_C4_KERNEL
).register_warmup()
else:
_SPARSE_ATTN_COMPRESS_C128_BLOCK8_KERNEL.register_warmup()
if store_full_kv:
_SPARSE_ATTN_NORM_ROPE_STORE_FULL_KERNEL.register_warmup()
else:
_SPARSE_ATTN_NORM_ROPE_STORE_KERNEL.register_warmup(
vllm_config,
k_cache_prefix=self.k_cache_prefix,
compress_ratio=self.compress_ratio,
)
else:
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( # noqa: E501
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL,
)
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL.register_warmup()
def forward(
self,
# [num_tokens, 2 * self.coff * self.head_dim]
kv_score: torch.Tensor,
# [num_tokens]
positions: torch.Tensor,
rotary_emb,
) -> None:
# Each of shape [num_tokens, coff * self.head_dim]
# input bf16, output are fp32
kv, score = kv_score.split(
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
)
# Get the metadata and handle dummy profiling run.
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if not isinstance(attn_metadata, dict):
return
state_metadata = cast(
CompressorMetadata, attn_metadata[self.state_cache.prefix]
)
token_to_req_indices = state_metadata.token_to_req_indices
slot_mapping = state_metadata.slot_mapping
num_actual = slot_mapping.shape[0]
block_table = state_metadata.block_table
block_size = state_metadata.block_size
# [num_blocks, block_size, kv_dim+score_dim], where kv_dim == score_dim
state_cache = self.state_cache.kv_cache
# kv_state stored in first half, score_state stored in second half
state_width = state_cache.shape[-1] // 2
pdl_kwargs = (
{}
if current_platform.is_rocm() or current_platform.is_xpu()
else {"launch_pdl": False}
)
# Store the KV and score (with fused APE addition) in the state.
# NOTE: PDL is disabled — both this kernel and the compress kernels
# below depend on preceding kernel outputs (kv/score from the cublas
# GEMM; state_cache from this kernel) but neither emits/waits on PDL
# grid dependency primitives, so launch_pdl=True caused a
# read-after-write race and non-deterministic output.
_SAVE_PARTIAL_STATES_KERNEL(
kv=kv,
score=score,
ape=self.ape,
positions=positions,
state_cache=state_cache,
slot_mapping=slot_mapping,
block_size=block_size,
state_width=state_width,
compress_ratio=self.compress_ratio,
pdl_kwargs=pdl_kwargs,
)
# A captured graph would replay the capture batch's skip decision.
if (
current_platform.is_cuda()
and self.head_dim == 512
and self.compress_ratio == 128
and not torch.cuda.is_current_stream_capturing()
and state_metadata.c128_boundary is False
):
return
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
# RoPE requirements (kernel applies forward GPT-J style rotation):
# - is_neox_style=False (interleaved pairs, NOT split-half)
# - cos_sin_cache layout: [max_pos, rope_head_dim] with first half cos,
# second half sin (per-pair, length rope_head_dim // 2 each)
# - applied to LAST rope_head_dim elements of head_dim
# - position used: (positions // compress_ratio) * compress_ratio
cos_sin_cache = rotary_emb.cos_sin_cache
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
kv_cache = k_cache_layer.kv_cache
# Plain-row V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
# fp8_ds_mla path uses the UE8M0 paged uint8 layout.
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
fp8_scale = (
getattr(k_cache_layer, "_flashinfer_fp8_kv_scale", None)
if store_full_fp8
else None
)
# cutedsl (head=512) accepts the full-cache flags; triton (indexer/AMD)
# does not, so the two callables have different signatures.
compress_norm_rope_store_fn: Any
if current_platform.is_cuda() and self.head_dim == 512:
from .nvidia.ops.sparse_attn_compress_cutedsl import (
_SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL,
)
# head=512 on CUDA always uses cutedsl, for both the fp8_ds_mla
# layout and the plain full-cache layout. The full-cache flags
# are consumed only here.
compress_norm_rope_store_fn = _SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL
extra_kwargs: dict[str, Any] = dict(
store_full_kv=store_full_kv,
store_full_fp8=store_full_fp8,
fp8_scale=fp8_scale,
)
elif self._use_two_stage_fused_compressor:
# head=512 cr>=128 (no overlap): two-pass split compressor on the
# prefill suffix, single-pass on the decode prefix.
assert state_metadata.num_decode_tokens is not None
compress_norm_rope_store_fn = compress_norm_rope_store_two_stage_triton
extra_kwargs = {
"num_decode_tokens": state_metadata.num_decode_tokens,
"compress_scratch": self._compress_scratch,
}
else:
# Indexer path (head_dim == 128) or non-CUDA GPUs (AMD, XPU, etc.).
compress_norm_rope_store_fn = compress_norm_rope_store_triton
extra_kwargs = {}
compress_norm_rope_store_fn(
state_cache=state_cache,
num_actual=num_actual,
token_to_req_indices=token_to_req_indices,
positions=positions,
slot_mapping=slot_mapping,
block_table=block_table,
block_size=block_size,
state_width=state_width,
cos_sin_cache=cos_sin_cache,
kv_cache=kv_cache,
k_cache_metadata=k_cache_metadata,
pdl_kwargs=pdl_kwargs,
head_dim=self.head_dim,
rope_head_dim=self.rope_head_dim,
compress_ratio=self.compress_ratio,
overlap=self.overlap,
use_fp4_cache=self.use_fp4_cache,
rms_norm_weight=self.norm.weight,
rms_norm_eps=self.rms_norm_eps,
quant_block=self._quant_block,
token_stride=self._token_stride,
scale_dim=self._scale_dim,
**extra_kwargs,
)
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 compressor subclass and dispatch: partial-state-cache
write and the compress -> RMSNorm -> RoPE -> FP8 quant -> KV cache store
step, calling ``torch.ops._C.save_partial_states_cpu``/
``compress_norm_rope_store_cpu``/``compress_norm_rope_store_indexer_cpu``
(``csrc/cpu/sgl-kernels/compressor.cpp``) in place of triton-cpu.
Covers both the head_dim=512 main-attention compressor path and the
head_dim=128 indexer compressor path (fp8 only -- MXFP4, ``use_fp4_cache``,
never occurs on CPU: the indexer's MXFP4 cache requires a Blackwell GPU).
"""
from typing import Any, cast
import torch
from vllm._custom_ops import (
compress_norm_rope_store_cpu,
compress_norm_rope_store_indexer_cpu,
save_partial_states_cpu,
)
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.compressor import CompressorMetadata, DeepseekCompressor
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
class DeepseekV4CPUCompressor(DeepseekCompressor):
"""CPU compressor: same state/weights as the shared base, but
``forward``'s cache-write and compress->RMSNorm->RoPE->quant->store
steps always dispatch straight to the ported CPU kernels above instead
of through the shared method's platform-dispatch chain. This class
only ever runs on CPU, and ``use_fp4_cache`` is always ``False`` here
(the indexer's MXFP4 cache requires a Blackwell GPU), so the shared
method's MXFP4/two-stage/cutedsl/triton branches never apply.
"""
_norm_weight_fp32: torch.Tensor
def cache_norm_weight_fp32(self) -> None:
"""Cache a fp32 contiguous copy of ``norm.weight`` once, right after
weight loading (see
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
kernel requires fp32/contiguous but the checkpoint loads this weight
in bf16, and it never changes after loading."""
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
def forward(
self,
kv_score: torch.Tensor,
positions: torch.Tensor,
rotary_emb,
) -> None:
kv, score = kv_score.split(
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
)
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return
state_metadata = attn_metadata[self.state_cache.prefix]
assert isinstance(state_metadata, CompressorMetadata)
token_to_req_indices = state_metadata.token_to_req_indices
assert token_to_req_indices is not None
slot_mapping = state_metadata.slot_mapping
num_actual = slot_mapping.shape[0]
block_table = state_metadata.block_table
block_size = state_metadata.block_size
state_cache = self.state_cache.kv_cache
save_partial_states_cpu(
kv, score, self.ape, positions, state_cache, slot_mapping
)
cos_sin_cache = rotary_emb.cos_sin_cache
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
kv_cache = k_cache_layer.kv_cache
assert self.head_dim in (512, 128), (
f"unsupported compressor head_dim: {self.head_dim}"
)
window = (1 + self.overlap) * self.compress_ratio
win_offsets = torch.arange(window, device=positions.device)
win_local = positions.unsqueeze(-1) - window + 1 + win_offsets
req_idx = token_to_req_indices[:num_actual]
gather_slots = map_local_to_global_slots_cpu(
win_local, req_idx, block_table, block_size
)
kv_cache_2d = kv_cache.view(kv_cache.shape[0], -1)
kv_cache_block_size = kv_cache.shape[1]
store_fn = (
compress_norm_rope_store_cpu
if self.head_dim == 512
else compress_norm_rope_store_indexer_cpu
)
store_fn(
state_cache,
gather_slots,
positions,
k_cache_metadata.slot_mapping,
self._norm_weight_fp32,
self.rms_norm_eps,
cos_sin_cache,
kv_cache_2d,
kv_cache_block_size,
self.compress_ratio,
)
-126
View File
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 sparse-MLA backend descriptor.
DeepSeek-V4 attention runs entirely through
``DeepseekV4Attention``/``DeepseekV4CPUAttention``, never through a generic
``AttentionImpl``. ``DeepseekV4CPUSparseBackend`` exists so the CPU attention
layer has its own backend name and its own metadata-builder class: CPU is not
allowed to run triton-cpu inside the model's forward path, so
``DeepseekV4CPUFlashMLAMetadataBuilder`` overrides the shared base's
``_build_c128a_metadata`` (triton-backed in
``DeepseekV4SparseMLAMetadataBuilder``) with a plain-eager-PyTorch
reimplementation of the same per-token block-table-resolution math.
"""
import torch
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4SparseMLABackend,
DeepseekV4SparseMLAMetadataBuilder,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend,
DeepseekV32IndexerMetadataBuilder,
)
from vllm.v1.attention.backends.mla.sparse_swa import (
_LAYER_TYPE_C4A,
_LAYER_TYPE_C128A,
_LAYER_TYPE_SWAONLY,
DeepseekSparseSWAMetadataBuilder,
)
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta
class DeepseekV4CPUFlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
"""CPU sparse-MLA metadata builder: same fields as the shared base, but
skips the C128A dense-topk metadata entirely -- unlike CUDA/XPU/ROCm,
``DeepseekV4CPUAttention.forward_mqa`` never reads
``c128a_global_decode_topk_indices``/``c128a_decode_topk_lens``/
``c128a_prefill_topk_indices``; it recomputes the same local top-k
directly from ``positions`` instead, so building it here (whether
eagerly or via the shared base's triton kernel) would be wasted work.
"""
def _build_c128a_metadata(
self,
cm: CommonAttentionMetadata,
req_id_per_token: torch.Tensor,
) -> dict[str, torch.Tensor | None]:
return {}
class DeepseekV4CPUSparseBackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "CPU_V4_MLA_SPARSE"
@staticmethod
def get_builder_cls() -> type[DeepseekV4CPUFlashMLAMetadataBuilder]:
return DeepseekV4CPUFlashMLAMetadataBuilder
class DeepseekV4CPUIndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder):
"""CPU indexer metadata builder: same fields as the shared base, but
prefill requests are never split into multiple chunks.
``DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks`` (the
shared base's default) bounds two things the CUDA/XPU indexer kernels
need: the flat-gather workspace size and the dense M*N logits tensor
those kernels allocate for a chunk. The CPU indexer
(``sparse_attn_indexer_cpu``) allocates neither -- it reads the paged
K-cache directly via ``fp8_paged_mqa_logits_cpu``/
``topk_transform_512_cpu``, with no chunk-splitting of its own.
With chunked prefill now enabled for this model on CPU (see
``CpuPlatform.check_and_update_config``'s ``amx_mla_or_dsv4_enabled``),
the total query-token count for one step is already bounded by
``max_num_batched_tokens`` -- there is nothing left here to bound, so
this always returns a single chunk spanning the whole step's prefill
batch.
"""
@staticmethod
def _split_indexer_prefill_chunks(
compressed_seq_lens_cpu: torch.Tensor,
prefill_query_lens_cpu: torch.Tensor,
workspace_size: int,
max_logits_bytes: int,
request_offset: int = 0,
) -> list[tuple[slice, slice]]:
# workspace_size/max_logits_bytes are unused: no flat-gather
# workspace or dense M*N buffer is allocated.
num_reqs = compressed_seq_lens_cpu.shape[0]
total_query_len = int(prefill_query_lens_cpu.sum().item())
return [
(
slice(request_offset, request_offset + num_reqs),
slice(0, total_query_len),
)
]
class DeepseekV4CPUIndexerBackend(DeepseekV4IndexerBackend):
@staticmethod
def get_name() -> str:
return "CPU_V4_INDEXER"
@staticmethod
def get_builder_cls() -> type[DeepseekV4CPUIndexerMetadataBuilder]:
return DeepseekV4CPUIndexerMetadataBuilder
class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
"""CPU SWA metadata builder: same fields as the shared base."""
def build_tile_scheduler(
self, num_decode_tokens: int
) -> dict[str, FlashMLASchedMeta | None]:
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
C++ decode-path concern) -- always return the all-``None`` sentinel
the shared base's own CPU branch returns."""
return {
_LAYER_TYPE_SWAONLY: None,
_LAYER_TYPE_C4A: None,
_LAYER_TYPE_C128A: None,
}
-649
View File
@@ -1,649 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 attention subclass.
``forward_mqa`` resolves SWA/compressed top-k indices to paged-cache slot
ids and calls the fused ``flash_mla_with_kvcache_cpu`` kernel. ``_o_proj``
stays eager -- not on the attention hot path.
"""
from typing import Any, cast
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import (
DeepseekV4Attention,
DeepseekV4Indexer,
DeepseekV4IndexerCache,
)
from vllm.models.deepseek_v4.cpu.cpu_compressor import DeepseekV4CPUCompressor
from vllm.models.deepseek_v4.cpu.cpu_mla import (
DeepseekV4CPUIndexerBackend,
DeepseekV4CPUSparseBackend,
)
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_ROPE_DIM = 64
def _deepseek_v4_cpu_prepare_and_attn(
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor | None,
indexer_kv_score: torch.Tensor | None,
indexer_weights: torch.Tensor | None,
positions: torch.Tensor,
o_padded: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
"""Opaque custom-op boundary around attention-prep + sparse-indexer +
MLA attention + output projection -- CPU-only.
Wraps cache writes with no explicit tensor args (SWA/compressor/indexer
state), which torch.compile can't otherwise track as mutations -- one
opaque call sidesteps that. ``_o_proj`` is folded in too because CPU's
``DYNAMO_TRACE_ONCE`` mode traces exactly once, during warmup when
``attn_metadata`` is ``None``; tracing ``_o_proj`` separately would bake
that warmup branch into the compiled graph permanently, dead-code-
eliminating the whole attention computation upstream.
"""
layer_name_str = _resolve_layer_name(layer_name)
self = get_forward_context().no_compile_layers[layer_name_str]
self._prepare_and_attn_fn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
)
o = o_padded[:, : self.n_local_heads, :]
return self._o_proj(o, positions)
def _deepseek_v4_cpu_prepare_and_attn_fake(
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor | None,
indexer_kv_score: torch.Tensor | None,
indexer_weights: torch.Tensor | None,
positions: torch.Tensor,
o_padded: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
# hidden_states' last dim already equals _o_proj's output width, so we
# don't need to resolve layer_name (an opaque object) here.
return hidden_states.new_empty((hidden_states.shape[0], hidden_states.shape[-1]))
direct_register_custom_op(
op_name="deepseek_v4_cpu_prepare_and_attn",
op_func=_deepseek_v4_cpu_prepare_and_attn,
mutates_args=["o_padded"],
fake_impl=_deepseek_v4_cpu_prepare_and_attn_fake,
)
def _dequant_linear_weight(layer: torch.nn.Module) -> torch.Tensor:
"""Dequantize a linear layer's weight, including FP8 block scales.
``_o_proj`` reads ``wo_a.weight`` directly (not via
``quant_method.apply()``), so the per-block FP8 scale must be applied
here explicitly.
"""
weight = layer.weight
scale = getattr(layer, "weight_scale_inv", None)
if scale is None:
scale = getattr(layer, "weight_scale", None)
if scale is None:
return weight.float()
if scale.dtype == torch.float8_e8m0fnu:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
scale = _upcast_e8m0_to_fp32(scale)
scale = scale.to(torch.float32)
if scale.numel() == 1:
return weight.float() * scale
block_size = getattr(layer, "weight_block_size", None)
if block_size is None:
# Per-channel scale: one row-scale per output row.
return weight.float() * scale.view(-1, 1)
block_m, block_k = block_size
m, k = weight.shape
scale = scale.repeat_interleave(block_m, dim=0).repeat_interleave(block_k, dim=1)
return weight.float() * scale[:m, :k]
def _fused_indexer_q_rope_quant_cpu(
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""CPU-only equivalent of ``fused_indexer_q_rope_quant``'s FP8 path
(CPU never reaches its MXFP4 arm -- Blackwell-only). Returns
``(q_fp8, weights_out)``, folding the per-token q scale into
``index_weights``. ``positions``/``index_q_cos_sin_cache`` are already
contiguous int64/fp32, so no conversion is needed here.
"""
assert positions.ndim == 1
assert index_q.ndim == 3
assert index_q_cos_sin_cache.ndim == 2
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
index_q_fp8 = torch.empty_like(index_q, dtype=current_platform.fp8_dtype())
ops.fused_indexer_q_rope_quant_cpu(
positions,
index_q.to(torch.float32),
index_q_cos_sin_cache,
index_q_fp8,
index_weights.to(torch.float32),
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out,
)
return index_q_fp8, index_weights_out
class _NoOpEvent:
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
stream list, these events are constructed but never actually used."""
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
"""CPU indexer K-cache descriptor: same fields as the shared base, just
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV4CPUIndexerBackend
class DeepseekV4CPUIndexer(DeepseekV4Indexer):
"""CPU indexer: the C4A short-context fallback runs as eager PyTorch
instead of Triton. Never constructed directly -- ``__class__`` is
swapped onto an existing ``DeepseekV4Indexer`` instance post-construction.
"""
def forward(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
compressed_kv_score: torch.Tensor | None,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
qr_scale: torch.Tensor | None = None,
skip_compressor: bool = False,
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
"""CPU override: no aux streams, so wq_b_and_q_quant and the
compressor run straight-line instead of through
``maybe_execute_in_parallel``; the short-context check drops the
CUDA-only ``is_current_stream_capturing()`` guard (always False
here)."""
compressor = self.compressor
attn_metadata = get_forward_context().attn_metadata
if isinstance(attn_metadata, dict):
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
if indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens:
# Fewer candidates than topk: all are selected, but the K
# cache still needs to be built.
compressor(compressed_kv_score, positions, rotary_emb)
assert self.topk_indices_buffer is not None
num_tokens = (
indexer_metadata.num_decode_tokens
+ indexer_metadata.num_prefill_tokens
)
if num_tokens > 0:
self._fill_short_context_topk(positions, num_tokens)
return None, None, None
def wq_b_and_q_quant():
q = self._wq_b_proj(qr, qr_scale)
q = q.view(-1, self.n_head, self.head_dim)
return _fused_indexer_q_rope_quant_cpu(
positions,
q,
rotary_emb.cos_sin_cache,
indexer_weights,
self.softmax_scale,
self.n_head**-0.5,
)
# Order matches the base class's parallel-join contract: compressor
# must complete before indexer_op runs (skip_k_cache_insert=True).
q, weights = wq_b_and_q_quant()
compressor(compressed_kv_score, positions, rotary_emb)
# No q_scale: FP8-only path (MXFP4 needs Blackwell).
return q, None, weights
def _fill_short_context_topk(
self, positions: torch.Tensor, num_tokens: int
) -> None:
assert self.topk_indices_buffer is not None
device = positions.device
arange = torch.arange(self.topk_tokens, device=device)
num_compressed = torch.div(
positions[:num_tokens] + 1, self.compress_ratio, rounding_mode="floor"
)
valid = arange.unsqueeze(0) < num_compressed.unsqueeze(-1)
filled = torch.where(
valid, arange.unsqueeze(0), torch.full_like(arange.unsqueeze(0), -1)
)
self.topk_indices_buffer[:num_tokens, : self.topk_tokens] = filled.to(
self.topk_indices_buffer.dtype
)
class DeepseekV4CPUAttention(DeepseekV4Attention):
"""CPU sparse MLA attention layer for DeepSeek V4."""
backend_cls = DeepseekV4CPUSparseBackend
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
# Swap classes post-construction (no class-swap hook on the shared
# base) so forward()/get_attn_backend() dispatch to CPU kernels.
# SWA-only layers (compress_ratio <= 1) never build a
# compressor/indexer.
if self.compressor is not None:
self.compressor.__class__ = DeepseekV4CPUCompressor
if self.indexer is not None:
self.indexer.__class__ = DeepseekV4CPUIndexer
self.indexer.k_cache.__class__ = DeepseekV4CPUIndexerCache
self.indexer.compressor.__class__ = DeepseekV4CPUCompressor
self._wo_a_packed: torch.Tensor | None = None
self._wrap_wo_a_process_weights_after_loading()
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
"""Cache fp32-contiguous copies of tensors CPU kernels want that way
but that arrive bf16 (rotary cos/sin table, compressor RMSNorm
weights) -- cast once here instead of per forward call.
Runs after every quantized layer's own
``process_weights_after_loading`` (see ``is_deferred_attention_layer``);
``wo_a``'s packing must happen *before* that phase instead, so it's a
separate monkeypatch in ``__init__``.
"""
self.rotary_emb.cos_sin_cache = self.rotary_emb.cos_sin_cache.to(
torch.float32
).contiguous()
if self.compressor is not None:
self.compressor.cache_norm_weight_fp32()
if self.indexer is not None:
self.indexer.compressor.cache_norm_weight_fp32()
def _wrap_wo_a_process_weights_after_loading(self) -> None:
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
``bmm_cpu`` before the FP8 kernel's own
``process_weights_after_loading`` VNNI-repacks it in place for
row-major reads ``_o_proj`` never does."""
wo_a = self.wo_a
quant_method = wo_a.quant_method
orig_pwal = quant_method.process_weights_after_loading
heads_per_group = self.n_local_heads // self.n_local_groups
k_dim = heads_per_group * self.head_dim
def _capture_and_pack(layer: torch.nn.Module) -> None:
dequant = _dequant_linear_weight(layer).to(torch.bfloat16)
dequant = dequant.view(self.n_local_groups, self.o_lora_rank, k_dim)
self._wo_a_packed = torch.ops._C.convert_weight_packed(dequant.contiguous())
orig_pwal(layer)
# orig_pwal VNNI-repacks layer.weight/scale for ordinary-linear
# use that _o_proj never needs -- free them, same pattern as
# dispatch_cpu_unquantized_gemm's remove_weight=True.
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
scale_attr = (
"weight_scale_inv"
if hasattr(layer, "weight_scale_inv")
else "weight_scale"
)
if hasattr(layer, scale_attr):
setattr(
layer,
scale_attr,
torch.nn.Parameter(torch.empty(0), requires_grad=False),
)
quant_method.process_weights_after_loading = ( # type: ignore[method-assign]
_capture_and_pack
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
"""CPU override: wraps attention-prep + MLA + output projection in the
CPU-only opaque custom op (see ``_deepseek_v4_cpu_prepare_and_attn``).
"""
num_tokens = hidden_states.shape[0]
o_padded = torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
kv_score: torch.Tensor | None = None
if self.compressor is not None:
# Must go through the module's own forward (not a raw matmul on
# its weight): the packed-GEMM kernel frees the raw weight after
# packing, same as wo_a's.
kv_score = self.compressor.fused_wkv_wgate(hidden_states).to(torch.float32)
indexer_weights: torch.Tensor | None = None
indexer_kv_score: torch.Tensor | None = None
if self.indexer is not None:
indexer_weights, _ = self.indexer.weights_proj(hidden_states)
indexer_kv_score = self.indexer.compressor.fused_wkv_wgate(
hidden_states
).to(torch.float32)
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
return torch.ops.vllm.deepseek_v4_cpu_prepare_and_attn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
_encode_layer_name(self.prefix),
)
def _fused_qnorm_rope_kv_insert(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
attn_metadata: (
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
),
) -> torch.Tensor:
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
supported here, so the base method's bf16/per-tensor-fp8 branches
are dropped."""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
if self.n_local_heads < self.padded_heads:
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
return q
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_kv_cache = self.swa_cache_layer.kv_cache
# positions is already int64 (the runner's buffer), no cast needed.
assert positions.dtype == torch.int64
assert swa_kv_cache.dtype == torch.uint8, (
"DeepseekV4CPUAttention only supports the fp8_ds_mla cache layout"
)
# Fused: Q side does per-head weight-free RMSNorm + GPT-J RoPE
# (zero-filling padded head slots); KV side does GPT-J RoPE + UE8M0
# FP8 quant + paged insert.
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
return ops.fused_qnorm_rope_kv_insert_cpu(
q,
kv,
positions,
swa_kv_cache_2d,
swa_metadata.slot_mapping,
self.rotary_emb.cos_sin_cache,
self.padded_heads,
self.eps,
swa_metadata.block_size,
)
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""CPU override: two ``RMSNorm`` calls instead of the shared
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
return self.q_norm(qr), None, self.kv_norm(kv)
def _prepare_and_attn(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
) -> None:
"""CPU override: no aux streams, so query-projection+KV-insert, the
indexer, and the compressor run straight-line instead of through
execute_in_parallel/maybe_execute_in_parallel."""
attn_metadata = get_forward_context().attn_metadata
indexer = self.indexer
compressor = self.compressor
q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_local_heads, self.head_dim)
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
index_q: torch.Tensor | None = None
index_q_scale: torch.Tensor | None = None
index_weights_out: torch.Tensor | None = None
if indexer is not None:
assert compressor is not None
index_q, index_q_scale, index_weights_out = indexer(
hidden_states,
qr,
indexer_kv_score,
indexer_weights,
positions,
self.indexer_rotary_emb,
qr_scale,
)
compressor(kv_score, positions, self.rotary_emb)
elif compressor is not None:
compressor(kv_score, positions, self.rotary_emb)
self._sparse_indexer_and_attn(
hidden_states,
index_q,
index_q_scale,
index_weights_out,
q,
kv,
positions,
o_padded,
)
def _sparse_indexer_and_attn(
self,
hidden_states: torch.Tensor,
index_q: torch.Tensor | None,
index_q_scale: torch.Tensor | None,
index_weights: torch.Tensor | None,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor,
) -> None:
"""CPU override: identical body, minus ``@eager_break_during_capture``
(this platform never captures a CUDA graph, so it was a no-op)."""
if self.indexer is not None and index_q is not None:
assert index_weights is not None
# index_q_scale is always None on CPU (FP8-only path).
assert index_q_scale is None
self.indexer.indexer_op(hidden_states, index_q, None, index_weights)
self.forward_mqa(q, kv, positions, out)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
# No head-count padding constraint for the (not yet ported) CPU kernel.
return num_heads
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
if get_forward_context().attn_metadata is None:
# Warmup/profile run: no real metadata, o is a zeroed placeholder.
return torch.zeros(
(o.shape[0], self.hidden_size), dtype=o.dtype, device=o.device
)
num_tokens, num_heads, _ = o.shape
cos_sin_cache = self.rotary_emb.cos_sin_cache
# Undo the query's own GPT-J RoPE rotation: the attention output is
# a weighted sum of KV rows at different positions, so R(-pos_q)
# recovers the translation-invariant quantity wo_a/wo_b expect.
o_derot = ops.inverse_gptj_rope_o_proj_cpu(
o, positions, cos_sin_cache, _ROPE_DIM
)
heads_per_group = num_heads // self.n_local_groups
o_grouped = o_derot.reshape(
num_tokens, self.n_local_groups, heads_per_group * self.head_dim
).to(torch.bfloat16)
assert self._wo_a_packed is not None
z = o_grouped.new_empty((self.n_local_groups, num_tokens, self.o_lora_rank))
ops.bmm_cpu(
z,
o_grouped.transpose(0, 1).contiguous(),
self._wo_a_packed,
True,
None,
)
z = z.transpose(0, 1).to(o.dtype)
return self.wo_b(z.flatten(1))
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# Warmup/profile run: reserve nothing extra, skip real kernels.
output.zero_()
return
assert isinstance(attn_metadata, dict)
swa_metadata = cast(
DeepseekSparseSWAMetadata,
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
assert swa_metadata.token_to_req_indices is not None
req_idx = swa_metadata.token_to_req_indices
swa_only = self.compress_ratio <= 1
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
num_tokens = q.shape[0]
device = q.device
# ---- SWA window: causal sliding window over the raw per-token cache
window = self.window_size
win_offsets = torch.arange(window, device=device)
start_pos = (positions - window + 1).clamp(min=0)
local_pos = start_pos.unsqueeze(-1) + win_offsets
valid_win = local_pos <= positions.unsqueeze(-1)
win_local = torch.where(valid_win, local_pos, torch.full_like(local_pos, -1))
window_slots = map_local_to_global_slots_cpu(
win_local, req_idx, swa_metadata.block_table, swa_metadata.block_size
)
window_cache_2d = self.swa_cache_layer.kv_cache.view(
self.swa_cache_layer.kv_cache.shape[0], -1
)
# ---- Compressed KV: C4A's learned top-k, or C128A's dense top-k ----
if swa_only:
compressed_slots = window_slots.new_full((num_tokens, 0), -1)
# Unused when num_compressed == 0 (only dtype/2D-ness matter);
# reuse the window cache to avoid an allocation.
compressed_cache_2d = window_cache_2d
compressed_block_size = 1
else:
assert flashmla_metadata is not None
main_block_size = flashmla_metadata.block_size // self.compress_ratio
if self.compress_ratio == 4:
assert self.topk_indices_buffer is not None
local_topk = self.topk_indices_buffer[:num_tokens]
else:
width = -(-self.max_model_len // self.compress_ratio)
num_compressed = torch.div(
positions + 1, self.compress_ratio, rounding_mode="floor"
).clamp(max=width)
arange_w = torch.arange(width, device=device)
local_topk = torch.where(
arange_w.unsqueeze(0) < num_compressed.unsqueeze(-1),
arange_w.unsqueeze(0).expand(num_tokens, width),
torch.full(
(num_tokens, width), -1, device=device, dtype=torch.int64
),
)
compressed_slots = map_local_to_global_slots_cpu(
local_topk, req_idx, flashmla_metadata.block_table, main_block_size
)
compressed_cache_2d = self.kv_cache.view(self.kv_cache.shape[0], -1)
compressed_block_size = main_block_size
torch.ops._C.flash_mla_with_kvcache_cpu(
output,
q,
window_cache_2d,
window_slots,
swa_metadata.block_size,
compressed_cache_2d,
compressed_slots,
compressed_block_size,
self.attn_sink,
self.scale,
)
-35
View File
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for DeepSeek-V4's CPU-ported kernels."""
import torch
def map_local_to_global_slots_cpu(
local_indices: torch.Tensor,
req_idx: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
) -> torch.Tensor:
"""Map per-request-local (token or compressed-token) indices to global
paged-cache slot ids via a block table, mirroring
``compute_global_topk_indices_and_lens``.
Args:
local_indices: [N, K], -1 sentinel for invalid entries.
req_idx: [N], row into ``block_table`` for each of the N rows.
block_table: [num_reqs, max_blocks_per_seq].
block_size: tokens (or compressed tokens) per physical block.
Returns:
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
"""
valid = local_indices >= 0
safe_local = local_indices.clamp(min=0).to(torch.int64)
block_pos = torch.div(safe_local, block_size, rounding_mode="floor")
offset = safe_local % block_size
req_expand = req_idx.to(torch.int64).unsqueeze(-1).expand_as(local_indices)
block_num = block_table[req_expand, block_pos].to(torch.int64)
slot = block_num * block_size + offset
return torch.where(valid, slot, torch.full_like(slot, -1))
-18
View File
@@ -1,18 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 DSpark (speculative decoding) — deferred, not implemented.
``DSparkDeepseekV4ForCausalLM`` is only instantiated when the speculative-
decoding registry resolves it by name for a DSpark speculative config; a
plain (non-spec) DeepSeek-V4 CPU model never constructs this class. This
stub exists solely so ``vllm.models.deepseek_v4.cpu`` is import-clean.
"""
from vllm.config import VllmConfig
class DSparkDeepseekV4ForCausalLM:
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
raise NotImplementedError(
"DeepSeek-V4 speculative decoding (DSpark) is not implemented on CPU."
)
-900
View File
@@ -1,900 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
from itertools import islice
import regex as re
import torch
import torch.nn as nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
HAS_TILELANG_MHC,
HCHeadOp,
MHCFusedPostPreOp,
MHCPostOp,
MHCPreOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
SupportsEagle3,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
from vllm.models.deepseek_v4.cpu.cpu_sparse import DeepseekV4CPUAttention
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
class DeepseekV4MLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
swiglu_limit: float | None = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
is_sequence_parallel: bool = False,
prefix: str = "",
) -> None:
super().__init__()
# is_sequence_parallel: weights are replicated, no collective ops
# needed. Otherwise: standard TP with an allreduce at the end.
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
disable_tp=is_sequence_parallel,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
disable_tp=is_sequence_parallel,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
if swiglu_limit is not None:
self.act_fn = SiluAndMulWithClamp(swiglu_limit)
else:
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class DeepseekV4MoE(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.prefix = prefix
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
self.hidden_size = config.hidden_size
self.n_routed_experts = config.n_routed_experts
self.n_activated_experts = config.num_experts_per_tok
self.moe_intermediate_size = config.moe_intermediate_size
self.swiglu_limit = config.swiglu_limit
self.renormalize = config.norm_topk_prob
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
self.gate = GateLinear(
input_size=config.hidden_size,
output_size=config.n_routed_experts,
bias=False,
out_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
"hash_moe"
)
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
# Use randint instead of empty to avoid garbage values causing
# invalid memory access in dummy mode (--load-format="dummy")
self.gate.tid2eid = nn.Parameter(
torch.randint(
0,
config.n_routed_experts,
(config.vocab_size, config.num_experts_per_tok),
dtype=self.hash_indices_dtype,
),
requires_grad=False,
)
elif getattr(config, "topk_method", None) == "noaux_tc":
self.gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.n_routed_experts, dtype=torch.float32),
requires_grad=False,
)
self.n_shared_experts = config.n_shared_experts
# No CPU fused-shared-expert grouped-GEMM path yet.
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
self.shared_experts = DeepseekV4MLP(
hidden_size=config.hidden_size,
intermediate_size=intermediate_size,
hidden_act=config.hidden_act,
swiglu_limit=self.swiglu_limit,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.shared_experts",
)
self.tp_rank = get_tensor_model_parallel_rank()
assert config.n_routed_experts % self.tp_size == 0
self.n_local_experts = config.n_routed_experts // self.tp_size
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
n_shared_experts=(
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
prefix=f"{prefix}.experts",
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
e_score_correction_bias=self.gate.e_score_correction_bias,
hash_indices_table=self.gate.tid2eid,
swiglu_limit=self.swiglu_limit,
router_logits_dtype=torch.float32,
)
def forward(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
) -> torch.Tensor:
if self.gate.tid2eid is not None and input_ids is None:
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
org_shape = hidden_states.shape
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
input_ids=input_ids,
)
return final_hidden_states.view(org_shape)
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
vllm_config,
prefix,
topk_indices_buffer: torch.Tensor | None = None,
):
super().__init__()
# Lazy import to avoid top-level tilelang dependency.
# Registers both torch.ops.vllm.mhc_pre and mhc_post
import vllm.model_executor.layers.mhc # noqa: F401
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.rms_norm_eps = config.rms_norm_eps
self.attn = DeepseekV4CPUAttention(
vllm_config,
prefix=f"{prefix}.attn",
topk_indices_buffer=topk_indices_buffer,
)
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.hc_mult = config.hc_mult
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
self.hc_eps = config.hc_eps
self.hc_post_alpha = 2.0
mix_hc = (2 + self.hc_mult) * self.hc_mult
hc_dim = self.hc_mult * self.hidden_size
self.hc_attn_fn = nn.Parameter(
torch.empty(
(mix_hc, hc_dim),
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_fn = nn.Parameter(
torch.empty(
(mix_hc, hc_dim),
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_attn_base = nn.Parameter(
torch.empty(
mix_hc,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_base = nn.Parameter(
torch.empty(
mix_hc,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_attn_scale = nn.Parameter(
torch.empty(
3,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_scale = nn.Parameter(
torch.empty(
3,
dtype=torch.float32,
),
requires_grad=False,
)
self.mhc_pre = MHCPreOp()
self.mhc_post = MHCPostOp()
self.mhc_fused_post_pre = MHCFusedPostPreOp()
self.use_fused_mhc = HAS_TILELANG_MHC and not (
HAS_AITER_MHC and self.hidden_size % 256 == 0
)
def hc_pre(
self,
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
):
post_mix, res_mix, layer_input = self.mhc_pre(
residual=x,
fn=hc_fn,
hc_scale=hc_scale,
hc_base=hc_base,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
)
return layer_input, post_mix, res_mix
def hc_post(
self,
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
):
return self.mhc_post(x, residual, post, comb)
def _forward_fused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if residual is None:
# Run standalone hc_pre on first layer
residual = x
x, post_mix, res_mix = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
else:
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
def _forward_unfused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
]:
residual = x
x, post, comb = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x, None, None, None
def forward(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
]:
if not self.use_fused_mhc:
return self._forward_unfused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
return self._forward_fused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.vocab_size = config.vocab_size
self.hc_eps = config.hc_eps
self.hc_mult = config.hc_mult
self.hc_dim = self.hc_mult * config.hidden_size
self.rms_norm_eps = config.rms_norm_eps
self.device = current_platform.device_type
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=self.device,
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV4DecoderLayer(
vllm_config,
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV4MoE,
"ffn",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.hc_head_fn = nn.Parameter(
torch.empty(
self.hc_mult,
self.hc_dim,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(
self.hc_mult,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_op = HCHeadOp()
spec_config = vllm_config.speculative_config
needs_mtp_hidden_states = spec_config is not None and (
spec_config.use_eagle() or spec_config.uses_draft_model()
)
if get_pp_group().is_last_rank and needs_mtp_hidden_states:
self._mtp_hidden_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
self.hc_dim,
dtype=vllm_config.model_config.dtype,
device=self.device,
)
else:
self._mtp_hidden_buffer = None
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def make_empty_intermediate_tensors(
self,
batch_size: int,
dtype: torch.dtype,
device: torch.device,
) -> IntermediateTensors:
# PP intermediate tensors keep the (num_tokens, hc_mult, hidden_size)
# multi-stream shape until hc_head() collapses it.
return IntermediateTensors(
{
"hidden_states": torch.zeros(
(batch_size, self.hc_mult, self.config.hidden_size),
dtype=dtype,
device=device,
),
}
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
hidden_states = hidden_states.unsqueeze(-2).repeat(1, self.hc_mult, 1)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual, post_mix, res_mix = None, None, None
# EAGLE3/DSpark/DFlash aux hidden states: post-mhc hidden state at
# each target layer, averaged over hc_mult streams to [T,
# hidden_size]. Empty unless a draft model set aux_hidden_state_layers.
aux_hidden_states: list[torch.Tensor] = []
# Reused below to avoid computing hc_post twice if the last layer is
# also an aux layer.
final_aux_recon: torch.Tensor | None = None
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
)
if (idx + 1) in self.aux_hidden_state_layers:
# Unfused path already applied hc_post; fused path needs it
# applied here before averaging.
if layer.use_fused_mhc:
aux_recon = layer.hc_post(
hidden_states, residual, post_mix, res_mix
)
final_aux_recon = aux_recon
else:
aux_recon = hidden_states
aux_hidden_states.append(aux_recon.mean(dim=1))
if layer is not None and layer.use_fused_mhc:
# Reuse the last layer's hc_post output if already computed above.
if (
final_aux_recon is not None
and self.end_layer in self.aux_hidden_state_layers
):
hidden_states = final_aux_recon
else:
hidden_states = layer.hc_post(
hidden_states, residual, post_mix, res_mix
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
if self._mtp_hidden_buffer is not None:
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
hidden_states = self.hc_head_op(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
self.hc_head_base,
self.rms_norm_eps,
self.hc_eps,
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
("compressor.fused_wkv_wgate", "compressor.wkv", 0),
("compressor.fused_wkv_wgate", "compressor.wgate", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _resolve_param_name(name: str) -> str:
inv_name = f"{name}_inv"
if name not in params_dict and inv_name in params_dict:
return inv_name
return name
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
# Each MoE's own per-layer fusion decision is the source of truth,
# so the redirect below can't diverge from how it was built.
fuse_by_layer = {
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
for mod_name, mod in self.named_modules()
if isinstance(mod, DeepseekV4MoE)
}
n_routed = self.config.n_routed_experts
# Maps the single shared-expert tensor group to one appended slot;
# multiple shared experts would need per-expert slicing (see
# deepseek_v2.py). DeepSeek-V4 has n_shared_experts == 1.
if any(fuse_by_layer.values()) and self.config.n_shared_experts != 1:
raise NotImplementedError(
"deepseek-v4 fused shared-expert loading supports only "
f"n_shared_experts == 1, got {self.config.n_shared_experts}"
)
for name, loaded_weight in weights:
# Shared-expert fusion: redirect shared_experts.w{1,2,3} into
# the appended routed-expert slot so it loads through the
# routed expert (grouped GEMM) loader.
if ".ffn.shared_experts.w" in name and fuse_by_layer.get(
extract_layer_index(name), False
):
name = name.replace(
".ffn.shared_experts.w",
f".ffn.experts.{n_routed}.w",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
continue
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
break
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(param_name)
break
else:
if ".experts." in name:
# E8M0 scales are float8_e8m0fnu in checkpoints but the
# MoE param is uint8; copy_() would numerically convert
# (e.g. 2^-7 -> 0) instead of preserving raw bytes.
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for mapping in expert_mapping:
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
# return_success lets us skip experts that have
# another replica available.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
loaded_params.add(name_mapped)
continue
else:
if is_pp_missing_parameter(name, self):
continue
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(param_name)
continue
return loaded_params
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# (param_name, weight_name, expert_id, shard_id). When fusing shared
# experts, include the appended slots so redirected shared-expert
# weights route through the expert loader.
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.n_routed_experts + (
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=num_experts,
)
def _make_deepseek_v4_weights_mapper(
expert_dtype: str, fuse_shared_experts: bool = False
) -> WeightsMapper:
if expert_dtype == "fp4":
# MXFP4 experts register scales as ``w{1,2,3}_weight_scale`` (no
# _inv); FP8 linear/shared-expert scales register as
# ``weight_scale_inv``. Route DeepSeek-native ``.scale`` and
# AMD-Quark ``.weight_scale`` keys accordingly, leaving expert/
# shared-expert ``w{1,2,3}.weight_scale`` untouched for the MXFP4
# loader.
scale_regex = {
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
re.compile(r"\.scale$"): ".weight_scale_inv",
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
}
else:
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
# scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys
# there.
scale_regex = {
re.compile(r"\.scale$"): ".weight_scale_inv",
}
# Fused shared experts are redirected to routed expert slots elsewhere;
# leave their names untouched here.
substr_map: dict[str, str | None] = (
{}
if fuse_shared_experts
else {".shared_experts.w2": ".shared_experts.down_proj"}
)
# No MTP support on CPU yet: drop mtp.* weights (no model.mtp submodule).
substr_map["mtp."] = None
return WeightsMapper(
orig_to_new_prefix={
"layers.": "model.layers.",
"embed.": "model.embed.",
"norm.": "model.norm.",
"hc_head": "model.hc_head",
"mtp.": "model.mtp.",
},
orig_to_new_regex=scale_regex,
orig_to_new_suffix={
"head.weight": "lm_head.weight",
"embed.weight": "embed_tokens.weight",
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
".input_scale": ".input_scale_2",
},
orig_to_new_substr=substr_map,
)
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
model_cls = DeepseekV4Model
# Default mapper assumes the original FP4-expert checkpoint layout.
# Overridden per-instance in __init__ when expert_dtype != "fp4".
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
packed_modules_mapping = {
"gate_up_proj": ["w1", "w3"],
"fused_wqa_wkv": ["wq_a", "wkv"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
expert_dtype = getattr(config, "expert_dtype", "fp4")
self.model = self.model_cls(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype,
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
# No CPU AMX weight-packing hook yet.
pass
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
-19
View File
@@ -1,19 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 MTP (speculative decoding) — deferred, not implemented.
``DeepSeekV4MTP`` is only instantiated when
``vllm/model_executor/models/registry.py``'s speculative-decoding registry
resolves it by name for a speculative-decoding config; a plain (non-spec)
DeepSeek-V4 CPU model never constructs this class. This stub exists solely
so ``vllm.models.deepseek_v4.cpu`` is import-clean.
"""
from vllm.config import VllmConfig
class DeepSeekV4MTP:
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
raise NotImplementedError(
"DeepSeek-V4 speculative decoding (MTP) is not implemented on CPU."
)
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-600
View File
@@ -1,600 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark draft model for DeepSeek-V4 (semi-autoregressive speculative decoding).
See: qwen3_dspark.py for base architecture. This one is specialized to the DSV4 DSpark,
which reuses the target model's architecture similarly to MTP.
To implement non-causal attention, we leverage the sparse attention implementation to
include the future query tokens in the top-k indices for each query token.
"""
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
hc_head_fused_kernel_tilelang,
mhc_post_tilelang,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4MixtureOfExperts,
DeepseekV4Model,
DeepseekV4MoE,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
logger = init_logger(__name__)
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$")
def _duplicate_context_wkv_weights(
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
) -> Iterable[tuple[str, torch.Tensor]]:
"""Load every draft layer's WKV into the cross-layer projection."""
for name, weight in weights:
yield name, weight
match = _CONTEXT_WKV_RE.fullmatch(name)
if match is None:
continue
layer_idx = int(match.group(1))
if layer_idx >= num_layers:
continue
stacked_weight = weight.detach()
stacked_weight.shard_id = layer_idx
yield f"context_wkv_proj.{match.group(2)}", stacked_weight
class DSparkDeepseekV4Model(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
self.hidden_size = config.hidden_size
self.hc_mult = config.hc_mult
self.hc_eps = config.hc_eps
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
# Shared with the target (aliased by the speculator's loading utility).
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.main_proj = ReplicatedLinear(
config.hidden_size * len(self.target_layer_ids),
config.hidden_size,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "main_proj"),
)
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
)
current_vllm_config = get_current_vllm_config()
self.layers = nn.ModuleList(
[
DeepseekV4DecoderLayer(
current_vllm_config,
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
topk_indices_buffer=self.topk_indices_buffer,
)
for i in range(self.num_dspark_layers)
]
)
self.context_wkv_proj = MergedColumnParallelLinear(
config.hidden_size,
[config.head_dim] * self.num_dspark_layers,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "context_wkv_proj"),
disable_tp=True,
)
# Heads: final norm + hc_head, and the Markov + confidence heads
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32), requires_grad=False
)
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", True):
self.confidence_head = DSparkConfidenceHead(
config.hidden_size + config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "confidence_head"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
"""
return self.main_norm(self.main_proj(aux_hidden_states))
@torch.inference_mode()
def precompute_and_store_context_kv(
self,
main_x: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
"""Insert the sliding-window context KV for every draft layer.
Mirrors the reference DSparkAttention: each layer derives its context KV
from the SAME projected target hidden ``main_x``, via that layer's own
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
layer's context slots.
``context_slot_mappings`` is a per-layer list (each entry is the context
slot mapping for that layer's kv-cache group, since the hybrid manager may
place draft layers in different groups). ``None`` (or a ``None`` entry)
runs the projection to reserve workspace but writes nothing (profiling).
"""
all_kv = self.context_wkv_proj(main_x).view(
main_x.shape[0], self.num_dspark_layers, self.config.head_dim
)
for i, (layer, kv) in enumerate(
zip(self.layers, all_kv.unbind(1), strict=True)
):
slot_mapping = (
None if context_slot_mappings is None else context_slot_mappings[i]
)
attn = layer.attn
kv = attn.kv_norm(kv)
if slot_mapping is None:
continue
_insert_context_kv(attn, kv, context_positions, slot_mapping)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, inputs_embeds
)
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_native_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
residual = post_mix = res_mix = None
for layer in self.layers:
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
mega_gate_metadata=mega_gate_metadata,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
# hc_head reduces the hc copies; return the PRE-norm head hidden
hidden_states = hc_head_fused_kernel_tilelang(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
self.hc_head_base,
self.rms_norm_eps,
self.hc_eps,
)
return hidden_states
def _insert_context_kv(
attn: nn.Module,
kv: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
query and discard it, since context tokens have no query). Mirrors
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
"""
swa_cache = attn.swa_cache_layer.kv_cache
block_size = attn.swa_cache_layer.block_size
cos_sin_cache = attn.rotary_emb.cos_sin_cache
cache_dtype = swa_cache.dtype
n_ctx = kv.shape[0]
dummy_q = torch.zeros(
(n_ctx, attn.n_local_heads, attn.head_dim),
dtype=kv.dtype,
device=kv.device,
)
if cache_dtype == torch.uint8:
# fp8_ds_mla UE8M0 paged layout
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
dummy_q,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin_cache,
attn.padded_heads,
attn.eps,
block_size,
)
elif cache_dtype == torch.bfloat16:
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
dummy_q,
kv,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn.eps,
block_size,
)
else: # per-tensor fp8 (torch.float8_e4m3fn)
# TODO(ben): double-check if this is being dispatched correctly for FI backend
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
dummy_q,
kv,
dummy_q_fp8,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn._flashinfer_fp8_kv_scale,
attn._flashinfer_fp8_q_scale_inv,
attn.eps,
block_size,
)
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
# load_dspark_model always aliases the target's.
has_own_embed_tokens = False
has_own_lm_head = False
# Full-vocab draft: draft ids are target ids, no remapping needed.
draft_id_to_target_id = None
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
self.pad_shared_expert = getattr(
self.quant_config, "weight_block_size", None
) is not None and not _use_sequence_parallel(vllm_config)
self.model = DSparkDeepseekV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# Shared with the target (aliased by the speculator's load utility).
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
collect_moe_layers(
self,
self.model.layers,
self.config,
decoder_layer_type=DeepseekV4DecoderLayer,
moe_type=DeepseekV4MoE,
)
# --- Hooks used by the speculator -------------------------------------
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
return self.model.combine_hidden_states(aux_hidden_states)
def get_draft_kv_cache_layer_names(self) -> list[str]:
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
# layer, named by its prefix.
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
def precompute_and_store_context_kv(
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
self.model.precompute_and_store_context_kv(
context_states, context_positions, context_slot_mappings
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
# Returns the pre-norm hc_head hidden ([T, hidden_size]).
return self.model(input_ids, positions, inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Base logits U_k = lm_head(norm(head_hidden))."""
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Full-vocab draft: base logits, no d2t scatter.
return self.compute_logits(hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
return draft_ids # full-vocab: draft ids are target ids
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
# --- Weight loading ----------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
Non-mtp weights (embed/head/main layers) belong to the target model and
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
first_layer = self.model.layers[0]
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
if use_native_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
else:
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
num_redundant_experts=first_layer.ffn.n_redundant_experts,
)
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
stacked_params_mapping = [
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_confidence_head = False
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
continue
name = mapped
if "confidence_head." in name:
loaded_confidence_head = True
# ``.scale`` -> per-method scale suffix.
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
)
if name.startswith("model.context_wkv_proj."):
param = params_dict[name]
param.weight_loader(param, loaded_weight, loaded_weight.shard_id)
loaded_params.add(name)
continue
# E8M0 expert scales: keep raw exponent bytes.
if ".experts." in name:
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
success = param.weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(name_mapped)
break
continue
# Stacked rules only apply to decoder-layer weights. Head-stack params
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
param.weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
break
else:
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
Returns None for non-mtp weights (owned by the target model).
"""
if name.startswith("context_wkv_proj."):
return f"model.{name}"
m = re.match(r"mtp\.(\d+)\.(.*)", name)
if m is None:
return None
stage = int(m.group(1))
rest = m.group(2)
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
return None
# Head-stack params live at model level (mtp.last), context combiner at
# model level (mtp.0); everything else is a per-layer decoder block.
head_prefixes = (
"norm.",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
"markov_head.",
"confidence_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
):
return f"model.{rest}"
return f"model.layers.{stage}.{rest}"
@@ -1,986 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4 FlashInfer sparse MLA backend."""
from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import (
build_flashinfer_mixed_sparse_indices,
compute_global_topk_indices_and_lens,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
rope_quant_attn_out,
rope_quant_unsupported_reason,
)
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
DeepseekV4SparseMLAMetadataBuilder,
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
from vllm.v1.attention.backend import AttentionCGSupport, MultipleOf
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.backends.mla.sparse_swa import (
DeepseekSparseSWABackend,
DeepseekSparseSWAMetadataBuilder,
)
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
logger = init_logger(__name__)
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
_flashinfer_dsv4_workspace_by_device: dict[torch.device, torch.Tensor] = {}
def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
workspace = _flashinfer_dsv4_workspace_by_device.get(device)
if workspace is None:
workspace = torch.zeros(
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=device,
)
_flashinfer_dsv4_workspace_by_device[device] = workspace
return workspace
def _packed_block_span(pool: torch.Tensor) -> int:
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
block_size for unpacked KV, larger when packed (#44577). Raises if not
token-aligned."""
block_stride = pool.stride(0)
token_stride = pool.stride(-2)
if block_stride % token_stride != 0:
raise NotImplementedError(
"FLASHINFER_MLA_SPARSE_DSV4 packed KV requires the per-block stride "
f"({block_stride}) to be a multiple of the per-token stride "
f"({token_stride}); this layout is not supported yet."
)
return block_stride // token_stride
# Sparse MLA h_q counts accepted natively (flashinfer>=0.6.14, #3545).
_SPARSE_MLA_SUPPORTED_Q_HEADS = (8, 16, 32, 64, 128)
def _pad_to_supported_q_heads(num_heads: int) -> int:
for supported in _SPARSE_MLA_SUPPORTED_Q_HEADS:
if num_heads <= supported:
return supported
raise ValueError(
f"DeepseekV4 FlashInfer MLA Sparse does not support {num_heads} heads "
"(sparse MLA kernel requires h_q in {8, 16, 32, 64, 128})."
)
def _required_sm120_sparse_topk(vllm_config: VllmConfig, window_size: int) -> int:
"""Return the SM120 DSV4 SWA specialization needed by this model."""
if not vllm_config.attention_config.use_non_causal:
return window_size
speculative_config = vllm_config.speculative_config
if speculative_config is None:
return window_size
return get_dspark_swa_index_width(
window_size,
speculative_config.num_speculative_tokens,
)
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder``
"""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"bfloat16",
"fp8",
"fp8_e4m3",
"fp8_ds_mla",
]
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [256]
@staticmethod
def get_name() -> str:
return "FLASHINFER_MLA_SPARSE_DSV4"
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [512]
@classmethod
def supports_sink(cls) -> bool:
return True
@classmethod
def is_sparse(cls) -> bool:
return True
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major in [10, 12]
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
if device_capability.major == 10:
if kv_cache_dtype == "fp8_ds_mla":
return (
"FLASHINFER_MLA_SPARSE_DSV4 SM10x uses the plain "
"per-tensor FP8 KV layout, not fp8_ds_mla"
)
if kv_cache_dtype not in (None, "auto", "bfloat16", "fp8", "fp8_e4m3"):
return "kv_cache_dtype not supported"
return None
if device_capability.major == 12:
if kv_cache_dtype not in ("fp8", "fp8_e4m3", "fp8_ds_mla"):
return "kv_cache_dtype not supported"
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
if not has_flashinfer_sparse_mla_sm120():
return (
"FLASHINFER_MLA_SPARSE_DSV4 SM120 requires FlashInfer's "
"sparse MLA decode API"
)
return None
return "FLASHINFER_MLA_SPARSE_DSV4 requires SM10x or SM12x"
@staticmethod
def get_builder_cls() -> type["DeepseekV4FlashInferSparseMLAMetadataBuilder"]:
return DeepseekV4FlashInferSparseMLAMetadataBuilder
class DeepseekV4FlashInferSparseMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
"""Varlen-capable metadata builder for the FlashInfer sparse MLA backend."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
"""SWA metadata for the FlashInfer sparse decode path (varlen decode)."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
@staticmethod
def get_builder_cls() -> type[DeepseekSparseSWAFlashInferMetadataBuilder]:
return DeepseekSparseSWAFlashInferMetadataBuilder
class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
"""FlashInfer TRTLLM-gen sparse MLA attention layer for SM100 DeepSeek V4."""
backend_cls = DeepseekV4FlashInferMLASparseBackend
swa_backend_cls = DeepseekSparseSWAFlashInferBackend
use_fp8_ds_mla_layout: ClassVar[bool] = False
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(
self, o: torch.Tensor | QuantizedActivation, positions: torch.Tensor
) -> torch.Tensor:
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self.wo_b,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> torch.Tensor | QuantizedActivation:
if self._fuse_rope_quant:
return rope_quant_attn_out(num_tokens, hidden_states.device)
return super()._alloc_attn_out(num_tokens, hidden_states)
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
# RopeQuant derives each query's RoPE position as seq_len - q_len + i,
# which matches `positions` for DSpark's non-causal draft block too,
# since every call passes the real per-request seq_lens. The one
# exception is a draft block running past max_model_len, whose clamped
# positions it does not reproduce; that only costs those drafts.
reason = rope_quant_unsupported_reason(self)
self._fuse_rope_quant = reason is None
if reason is None:
logger.info_once(
"FLASHINFER_MLA_SPARSE_DSV4 fuses the inverse RoPE and FP8 "
"quant of the attention output."
)
else:
logger.debug_once("FLASHINFER_MLA_SPARSE_DSV4 RopeQuant off: %s", reason)
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
return
fp8_q_scale = 1.0
fp8_kv_scale = 1.0
self.register_buffer(
"_flashinfer_fp8_q_scale",
torch.tensor([fp8_q_scale], dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_flashinfer_fp8_q_scale_inv",
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_flashinfer_fp8_kv_scale",
torch.tensor([fp8_kv_scale], dtype=torch.float32),
persistent=False,
)
# TRTLLM-gen takes scalar scale args on a distinct C++ path vs
# one-element tensors, so these are Python floats.
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor | QuantizedActivation,
) -> None:
if isinstance(output, QuantizedActivation):
assert output.data.shape[0] == q.shape[0]
else:
# The TRTLLM-gen kernel requires h_q in {64, 128}, so the output
# buffer is allocated at the padded head count while q arrives at
# the local head count; _forward pads q to match before the launcher.
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
f"output buffer shape {output.shape} incompatible with q {q.shape}"
)
assert output.shape[1] >= q.shape[1], (
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
)
# Per-tensor FP8 q produces a bf16 attention output.
expected_output_dtype = (
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
)
assert output.dtype == expected_output_dtype, (
f"output dtype {output.dtype} must match expected "
f"{expected_output_dtype} for q dtype {q.dtype}"
)
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# Warmup dummy run: FlashInfer reads the cache directly and lazily
# allocates its workspace, so nothing to reserve here.
if isinstance(output, QuantizedActivation):
output.data.zero_()
output.scale.zero_()
else:
output.zero_()
return
assert isinstance(attn_metadata, dict)
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_only = self.compress_ratio <= 1
# SWA-only layers don't allocate their own compressed KV cache.
self_kv_cache = self.kv_cache if not swa_only else None
swa_kv_cache = self.swa_cache_layer.kv_cache
self._forward(
q=q,
kv_cache=self_kv_cache,
swa_k_cache=swa_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=flashmla_metadata,
swa_only=swa_only,
output=output,
)
def _build_sparse_index_metadata(
self,
kv_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
swa_metadata: "DeepseekSparseSWAMetadata",
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
num_rows: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build the combined sparse-index tensors for the mixed batch.
Returns ``(compressed_kv_cache, seq_lens, sparse_indices,
sparse_topk_lens)``, sized ``num_rows`` (default: the real tokens).
"""
num_decodes = swa_metadata.num_decodes
num_prefills = swa_metadata.num_prefills
num_decode_tokens = swa_metadata.num_decode_tokens
num_prefill_tokens = swa_metadata.num_prefill_tokens
num_reqs = num_decodes + num_prefills
num_tokens = num_decode_tokens + num_prefill_tokens
assert swa_metadata.seq_lens is not None
assert swa_metadata.query_start_loc is not None
assert swa_metadata.token_to_req_indices is not None
assert swa_metadata.decode_swa_indices is not None
assert swa_metadata.block_table is not None
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
num_decode_tokens, swa_metadata.decode_swa_width
)
decode_compressed_topk_lens = None
decode_compressed_indices_are_local = False
decode_is_valid_token = None
if swa_only:
assert self.topk_indices_buffer is not None
compressed_kv_cache = swa_k_cache
decode_compressed_indices = None
prefill_topk_indices = self.topk_indices_buffer[
num_decode_tokens:num_tokens, :0
]
compressed_block_table = None
compressed_block_size = swa_metadata.block_size
top_k = 0
else:
assert kv_cache is not None
assert attn_metadata is not None
compressed_kv_cache = kv_cache
compressed_block_table = attn_metadata.block_table[:num_reqs]
compressed_block_size = attn_metadata.block_size // self.compress_ratio
if self.compress_ratio == 4:
assert self.topk_indices_buffer is not None
if num_prefill_tokens > 0:
prefill_topk_indices = self.topk_indices_buffer[
num_decode_tokens:num_tokens
]
top_k = prefill_topk_indices.shape[-1]
else:
prefill_topk_indices = self.topk_indices_buffer[:0, :0]
top_k = 0
decode_compressed_indices_are_local = True
assert swa_metadata.is_valid_token is not None
decode_is_valid_token = swa_metadata.is_valid_token[:num_decode_tokens]
if num_decode_tokens > 0:
decode_compressed_indices = self.topk_indices_buffer[
:num_decode_tokens
]
else:
# Keep the logical width aligned with the mixed-batch case so
# pure-prefill steps reuse the same Triton specialization.
decode_compressed_indices = prefill_topk_indices[:0]
else:
if num_prefill_tokens > 0:
assert attn_metadata.c128a_prefill_topk_indices is not None
prefill_topk_indices = attn_metadata.c128a_prefill_topk_indices
top_k = prefill_topk_indices.shape[-1]
else:
prefill_topk_indices = decode_swa_indices[:0, :0]
top_k = 0
if num_decode_tokens > 0:
assert attn_metadata.c128a_global_decode_topk_indices is not None
assert attn_metadata.c128a_decode_topk_lens is not None
decode_compressed_indices = (
attn_metadata.c128a_global_decode_topk_indices.view(
num_decode_tokens, -1
)
)
decode_compressed_topk_lens = attn_metadata.c128a_decode_topk_lens
if num_prefill_tokens == 0:
prefill_topk_indices = decode_compressed_indices[:0, :0]
else:
decode_compressed_indices = prefill_topk_indices[:0]
decode_compressed_topk_lens = swa_metadata.seq_lens[:0]
query_start_loc = swa_metadata.query_start_loc[: num_reqs + 1]
seq_lens = swa_metadata.seq_lens[:num_reqs]
assert seq_lens.dtype == torch.int32
# cache for SWA-only and C128A that build the same mixed sparse indices
# C4A stays uncached.
cache_key = (
"swa_only"
if swa_only
else ("c128a" if self.compress_ratio == 128 else "c4a")
)
if num_rows is not None:
cache_key = f"{cache_key}:{num_rows}"
cached_sparse = swa_metadata.flashinfer_sparse_index_cache.get(cache_key, None)
if cached_sparse is None:
swa_block_span = _packed_block_span(swa_k_cache)
compressed_block_span = _packed_block_span(compressed_kv_cache)
sparse_indices, sparse_topk_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices,
decode_compressed_indices,
decode_compressed_topk_lens,
prefill_topk_indices[:num_prefill_tokens],
query_start_loc,
seq_lens,
swa_metadata.token_to_req_indices[:num_tokens],
swa_metadata.block_table[:num_reqs],
swa_metadata.block_size,
compressed_block_table,
compressed_block_size,
self.window_size,
self.compress_ratio,
top_k,
decode_compressed_indices_are_local=decode_compressed_indices_are_local,
decode_is_valid_token=decode_is_valid_token,
swa_block_span=swa_block_span,
compressed_block_span=compressed_block_span,
prefill_left_visible=swa_metadata.prefill_left_visible,
prefill_right_visible=swa_metadata.prefill_right_visible,
# getattr for tests that bypass __init__ via object.__new__.
max_image_tokens=getattr(self, "max_image_tokens", 0),
num_rows=num_rows,
)
if not cache_key.startswith("c4a"):
swa_metadata.flashinfer_sparse_index_cache[cache_key] = (
sparse_indices,
sparse_topk_lens,
)
else:
sparse_indices, sparse_topk_lens = cached_sparse
return compressed_kv_cache, seq_lens, sparse_indices, sparse_topk_lens
def _forward(
self,
q: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
swa_metadata: "DeepseekSparseSWAMetadata",
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
output: torch.Tensor | QuantizedActivation,
) -> None:
assert self.kv_cache_torch_dtype in (torch.bfloat16, torch.float8_e4m3fn)
num_decodes = swa_metadata.num_decodes
num_prefills = swa_metadata.num_prefills
num_decode_tokens = swa_metadata.num_decode_tokens
num_prefill_tokens = swa_metadata.num_prefill_tokens
num_reqs = num_decodes + num_prefills
num_tokens = num_decode_tokens + num_prefill_tokens
if num_tokens == 0:
return
# RopeQuant's output is group-major with a group stride of exactly
# sum_q tokens, so both calls span every row of the (CUDA-graph
# padded) buffers and select their tokens through cum_seq_lens_q
# alone; the kernel never touches rows outside it.
rope_quant = isinstance(output, QuantizedActivation)
(
compressed_kv_cache,
seq_lens,
sparse_indices,
sparse_topk_lens,
) = self._build_sparse_index_metadata(
kv_cache=kv_cache,
swa_k_cache=swa_k_cache,
swa_metadata=swa_metadata,
attn_metadata=attn_metadata,
swa_only=swa_only,
num_rows=q.shape[0] if rope_quant else None,
)
# CUDA graph execution can pad q/output past the scheduled token count;
# restrict to the real tokens (the launcher validates sparse indices).
query = q if rope_quant else q[:num_tokens]
bmm1_scale: float | torch.Tensor = self.scale
bmm2_scale: float | torch.Tensor = 1.0
if self.kv_cache_torch_dtype == torch.float8_e4m3fn:
assert query.dtype == torch.float8_e4m3fn
bmm1_scale = self._flashinfer_fp8_bmm1_scale
bmm2_scale = self._flashinfer_fp8_bmm2_scale
else:
assert query.dtype == torch.bfloat16
query = query.contiguous()
# The TRTLLM-gen sparse-MLA kernel requires h_q in {64, 128}; zero-pad
# the query heads to the allocated output head count. Padded heads attend
# to the shared KV and are sliced off downstream (output is padded too).
padded_heads = (
output.shape[1] if isinstance(output, torch.Tensor) else self.padded_heads
)
if query.shape[1] < padded_heads:
padded_query = query.new_zeros(
(query.shape[0], padded_heads, query.shape[2])
)
padded_query[:, : query.shape[1], :] = query
query = padded_query
workspace = _get_flashinfer_dsv4_workspace(q.device)
query_start_loc = swa_metadata.query_start_loc
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
assert query_start_loc is not None and query_start_loc_cpu is not None
def run(
rows: slice, reqs: slice, cum_seq_lens_q: torch.Tensor, max_q_len: int
) -> None:
if rope_quant:
assert isinstance(output, QuantizedActivation)
rows = slice(None)
out_kwargs = dict(
out=output.data,
dsv4_output_scale=output.scale,
dsv4_inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache,
)
else:
assert isinstance(output, torch.Tensor)
out_kwargs = dict(out=output[rows])
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
query=query[rows],
swa_kv_cache=swa_k_cache,
workspace_buffer=workspace,
sparse_indices=sparse_indices[rows],
sparse_indices_are_storage_offsets=True,
compressed_kv_cache=compressed_kv_cache,
sparse_topk_lens=sparse_topk_lens[rows],
seq_lens=seq_lens[reqs],
bmm1_scale=bmm1_scale,
bmm2_scale=bmm2_scale,
sinks=self.attn_sink,
cum_seq_lens_q=cum_seq_lens_q,
max_q_len=max_q_len,
**out_kwargs,
)
# Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for
# uniform-q batches, and this avoids flattening mixed batches into one call.
if num_decode_tokens > 0:
run(
slice(0, num_decode_tokens),
slice(0, num_decodes),
query_start_loc[: num_decodes + 1],
swa_metadata.max_decode_query_len,
)
if num_prefill_tokens > 0:
prefill_cu = query_start_loc[num_decodes : num_reqs + 1]
if not rope_quant:
# The prefill query view re-anchors at offset 0, so rebase the
# cumulative query offsets to start at 0.
prefill_cu = prefill_cu - query_start_loc[num_decodes]
prefill_cu_cpu = query_start_loc_cpu[num_decodes : num_reqs + 1]
prefill_lens_cpu = prefill_cu_cpu[1:] - prefill_cu_cpu[:-1]
run(
slice(num_decode_tokens, num_tokens),
slice(num_decodes, num_reqs),
prefill_cu,
int(prefill_lens_cpu.max().item()),
)
class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
"""DeepSeek V4 sparse MLA attention through FlashInfer's SM120 kernels."""
backend_cls = DeepseekV4FlashInferMLASparseBackend
swa_backend_cls = DeepseekSparseSWAFlashInferBackend
use_fp8_ds_mla_layout: ClassVar[bool] = True
@staticmethod
def _get_workspace(device: torch.device) -> torch.Tensor:
return _get_flashinfer_dsv4_workspace(device)
@staticmethod
def _as_sparse_cache(kv_cache: torch.Tensor) -> torch.Tensor:
if kv_cache.dtype == torch.float8_e4m3fn:
kv_cache = kv_cache.view(torch.uint8)
if kv_cache.dim() == 4:
return kv_cache
return kv_cache.unsqueeze(-2)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self.wo_b,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120_config
required_topk = _required_sm120_sparse_topk(vllm_config, self.window_size)
if not has_flashinfer_sparse_mla_sm120_config(self.padded_heads, required_topk):
raise RuntimeError(
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires a FlashInfer "
"DSV4 sparse MLA decode specialization for "
f"(num_q_heads={self.padded_heads}, top_k={required_topk}). "
"Install a FlashInfer build containing "
"flashinfer-ai/flashinfer#4380."
)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
# Per-tensor FP8 cache path scales.
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
return
fp8_q_scale = 1.0
fp8_kv_scale = 1.0
self.register_buffer(
"_flashinfer_fp8_q_scale",
torch.tensor([fp8_q_scale], dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_flashinfer_fp8_q_scale_inv",
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_flashinfer_fp8_kv_scale",
torch.tensor([fp8_kv_scale], dtype=torch.float32),
persistent=False,
)
# FlashInfer expects scalar scale arguments for this path.
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
def _reserve_empty_forward_workspace(self) -> None:
self._get_workspace(
torch.device("cuda", torch.accelerator.current_device_index())
)
def _forward_sparse_impl(
self,
q: torch.Tensor,
output: torch.Tensor,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
self_kv_cache: torch.Tensor | None,
swa_kv_cache: torch.Tensor,
swa_only: bool,
) -> None:
num_decode_tokens = swa_metadata.num_decode_tokens
if swa_metadata.num_prefills > 0:
self._forward_prefill(
q=q[num_decode_tokens:],
compressed_k_cache=self_kv_cache,
swa_k_cache=swa_kv_cache,
output=output[num_decode_tokens:],
attn_metadata=flashmla_metadata,
swa_metadata=swa_metadata,
)
if swa_metadata.num_decodes > 0:
self._forward_decode(
q=q[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=flashmla_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
)
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
# Output may be padded to backend-supported head counts.
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
f"output buffer shape {output.shape} incompatible with q shape {q.shape}"
)
assert output.shape[1] >= q.shape[1], (
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
)
# Per-tensor FP8 q produces a bf16 attention output.
expected_output_dtype = (
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
)
assert output.dtype == expected_output_dtype, (
f"output dtype {output.dtype} must match expected {expected_output_dtype} "
f"for q dtype {q.dtype}"
)
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
self._reserve_empty_forward_workspace()
output.zero_()
return
assert isinstance(attn_metadata, dict)
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_only = self.compress_ratio <= 1
# SWA-only layers don't allocate their own compressed KV cache.
self_kv_cache = self.kv_cache if not swa_only else None
swa_kv_cache = self.swa_cache_layer.kv_cache
self._forward_sparse_impl(
q=q,
output=output,
flashmla_metadata=flashmla_metadata,
swa_metadata=swa_metadata,
self_kv_cache=self_kv_cache,
swa_kv_cache=swa_kv_cache,
swa_only=swa_only,
)
def _prepare_query(self, q: torch.Tensor, output: torch.Tensor) -> torch.Tensor:
if self.kv_cache_torch_dtype == torch.float8_e4m3fn:
assert q.dtype == torch.float8_e4m3fn
q = q.to(torch.bfloat16)
else:
assert q.dtype == torch.bfloat16
padded_heads = output.shape[1]
if q.shape[1] < padded_heads:
padded_query = q.new_zeros((q.shape[0], padded_heads, q.shape[2]))
padded_query[:, : q.shape[1], :] = q
q = padded_query
return q.contiguous()
def _forward_decode(
self,
q: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_metadata: "DeepseekSparseSWAMetadata",
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
output: torch.Tensor,
) -> None:
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
extra_sparse_indices = None
extra_sparse_lengths = None
if not swa_only:
if attn_metadata is None:
raise RuntimeError(
"Sparse MLA metadata is required for compressed layers."
)
if swa_metadata.is_valid_token is None:
raise RuntimeError(
"SWA validity metadata is required for compressed layers."
)
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
if self.compress_ratio == 4:
if self.topk_indices_buffer is None:
raise RuntimeError(
"C4A decode requires top-k indices from the indexer."
)
block_size = attn_metadata.block_size // self.compress_ratio
global_indices, extra_sparse_lengths = (
compute_global_topk_indices_and_lens(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
block_size,
is_valid,
)
)
extra_sparse_indices = global_indices.view(num_decode_tokens, 1, -1)
else:
extra_sparse_indices = attn_metadata.c128a_global_decode_topk_indices
extra_sparse_lengths = attn_metadata.c128a_decode_topk_lens
swa_indices = swa_metadata.decode_swa_indices
swa_lens = swa_metadata.decode_swa_lens
assert swa_indices is not None
assert swa_lens is not None
q = self._prepare_query(q, output)
swa_cache = self._as_sparse_cache(self.swa_cache_layer.kv_cache)
extra_cache = self._as_sparse_cache(kv_cache) if kv_cache is not None else None
if extra_cache is not None and extra_sparse_indices is None:
raise RuntimeError(
"Compressed sparse MLA decode requires compressed sparse indices."
)
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
query=q,
swa_kv_cache=swa_cache,
workspace_buffer=self._get_workspace(q.device),
sparse_indices=swa_indices,
compressed_kv_cache=extra_cache,
out=output,
bmm1_scale=self.scale,
sinks=self.attn_sink,
kv_layout="NHD",
swa_topk_lens=swa_lens,
extra_sparse_indices=extra_sparse_indices,
extra_sparse_topk_lens=extra_sparse_lengths,
)
def _forward_prefill(
self,
q: torch.Tensor,
compressed_k_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
output: torch.Tensor,
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
) -> None:
swa_only = self.compress_ratio <= 1
num_prefills = swa_metadata.num_prefills
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
num_prefill_tokens = swa_metadata.num_prefill_tokens
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
assert query_start_loc_cpu is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
local_topk_indices: torch.Tensor | None
if swa_only:
local_topk_indices = None
elif self.compress_ratio == 4:
if self.topk_indices_buffer is None:
raise RuntimeError(
"C4A prefill requires top-k indices from the indexer."
)
local_topk_indices = self.topk_indices_buffer[
num_decode_tokens : num_decode_tokens + num_prefill_tokens
]
else:
if attn_metadata is None:
raise RuntimeError("C128A prefill metadata is missing.")
local_topk_indices = attn_metadata.c128a_prefill_topk_indices
extra_sparse_indices: torch.Tensor | None = None
extra_sparse_lengths: torch.Tensor | None = None
if local_topk_indices is not None:
if attn_metadata is None:
raise RuntimeError("C4A prefill metadata is missing.")
if swa_metadata.token_to_req_indices is None:
raise RuntimeError("C4A prefill request mapping is missing.")
if swa_metadata.is_valid_token is None:
raise RuntimeError("C4A prefill validity metadata is missing.")
prefill_token_slice = slice(
num_decode_tokens, num_decode_tokens + num_prefill_tokens
)
block_size = attn_metadata.block_size // self.compress_ratio
extra_sparse_indices, extra_sparse_lengths = (
compute_global_topk_indices_and_lens(
local_topk_indices,
swa_metadata.token_to_req_indices[prefill_token_slice],
attn_metadata.block_table,
block_size,
swa_metadata.is_valid_token[prefill_token_slice],
)
)
assert swa_metadata.prefill_swa_indices is not None
assert swa_metadata.prefill_swa_lens is not None
q = self._prepare_query(q, output)
swa_kv_paged = self._as_sparse_cache(swa_k_cache)
if swa_only:
extra_kv_paged = None
else:
if compressed_k_cache is None:
raise RuntimeError(
"Compressed sparse MLA layers require their compressed KV cache."
)
extra_kv_paged = self._as_sparse_cache(compressed_k_cache)
num_chunks = (
num_prefills + self.PREFILL_CHUNK_SIZE - 1
) // self.PREFILL_CHUNK_SIZE
for chunk_idx in range(num_chunks):
chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE
chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills)
query_start = (
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
)
query_end = (
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
)
extra_sparse_indices_chunk = (
extra_sparse_indices[query_start:query_end]
if extra_sparse_indices is not None
else None
)
extra_sparse_lengths_chunk = (
extra_sparse_lengths[query_start:query_end]
if extra_sparse_lengths is not None
else None
)
q_chunk = q[query_start:query_end]
swa_indices_chunk = swa_metadata.prefill_swa_indices[query_start:query_end]
swa_lens_chunk = swa_metadata.prefill_swa_lens[query_start:query_end]
if extra_kv_paged is not None and extra_sparse_indices_chunk is None:
raise RuntimeError(
"Compressed sparse MLA prefill requires compressed sparse indices."
)
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
query=q_chunk,
swa_kv_cache=swa_kv_paged,
workspace_buffer=self._get_workspace(q.device),
sparse_indices=swa_indices_chunk,
compressed_kv_cache=extra_kv_paged,
out=output[query_start:query_end],
bmm1_scale=self.scale,
sinks=self.attn_sink,
kv_layout="NHD",
swa_topk_lens=swa_lens_chunk,
extra_sparse_indices=extra_sparse_indices_chunk,
extra_sparse_topk_lens=extra_sparse_lengths_chunk,
)
-398
View File
@@ -1,398 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
)
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.mla.sparse_swa import (
DeepseekSparseSWABackend,
DeepseekSparseSWAMetadataBuilder,
)
from vllm.v1.attention.ops.flashmla import (
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
)
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
"""SWA metadata for the FlashMLA decode path, which allows varlen decode."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend):
@staticmethod
def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]:
return DeepseekSparseSWAFlashMLAMetadataBuilder
class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
"""FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA)."""
backend_cls = DeepseekV4FlashMLABackend
swa_backend_cls = DeepseekSparseSWAFlashMLABackend
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self.wo_b,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
# FP8 decode kernel only supports h_q = 64 or 128.
if num_heads > 128:
raise ValueError(
f"DeepseekV4 FlashMLA does not support {num_heads} heads "
"(FP8 decode kernel requires h_q in {64, 128})."
)
return 64 if num_heads <= 64 else 128
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
assert output.shape == q.shape, (
f"output buffer shape {output.shape} must match q shape {q.shape}"
)
assert output.dtype == q.dtype, (
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
)
# Get SWA and indexer metadata from forward context
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# Warmup dummy run: no real metadata. Reserve the same bf16
# gather workspace _forward_prefill would; the dequantize / topk
# / sparse_fwd kernels are skipped this step.
swa_only = self.compress_ratio <= 1
N = (
0
if swa_only
else (self.max_model_len + self.compress_ratio - 1)
// self.compress_ratio
)
M = N + self.window_size + self.max_num_batched_tokens
if swa_only:
top_k = 0
else:
assert self.topk_indices_buffer is not None
top_k = self.topk_indices_buffer.shape[-1]
combined_topk = round_up(
top_k + self.window_size + self.max_image_tokens, 128
)
current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
output.zero_()
return
assert isinstance(attn_metadata, dict)
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_only = self.compress_ratio <= 1
# SWA-only layers (compress_ratio <= 1) don't have their own KV cache
# allocation, so self.kv_cache may be empty after profiling cleanup.
self_kv_cache = self.kv_cache if not swa_only else None
swa_kv_cache = self.swa_cache_layer.kv_cache
# Split prefill and decode
num_decodes = swa_metadata.num_decodes
num_prefills = swa_metadata.num_prefills
num_decode_tokens = swa_metadata.num_decode_tokens
if num_prefills > 0:
self._forward_prefill(
q=q[num_decode_tokens:],
positions=positions[num_decode_tokens:],
compressed_k_cache=self_kv_cache,
swa_k_cache=swa_kv_cache,
output=output[num_decode_tokens:],
attn_metadata=flashmla_metadata,
swa_metadata=swa_metadata,
)
if num_decodes > 0:
self._forward_decode(
q=q[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=flashmla_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
)
def _forward_decode(
self,
q: torch.Tensor,
kv_cache: torch.Tensor | None, # Only used when compress_ratio > 1
swa_metadata: "DeepseekSparseSWAMetadata",
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
output: torch.Tensor,
) -> None:
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
topk_indices = None
topk_lens = None
if not swa_only:
assert attn_metadata is not None
assert swa_metadata.is_valid_token is not None
block_size = attn_metadata.block_size // self.compress_ratio
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
if self.compress_ratio == 4:
# C4A: local indices differ per layer (filled by Indexer).
assert self.topk_indices_buffer is not None
global_indices, topk_lens = compute_global_topk_indices_and_lens(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
block_size,
is_valid,
)
topk_indices = global_indices.view(num_decode_tokens, 1, -1)
else:
# C128A: pre-computed during metadata build.
topk_indices = attn_metadata.c128a_global_decode_topk_indices
topk_lens = attn_metadata.c128a_decode_topk_lens
swa_indices = swa_metadata.decode_swa_indices
swa_lens = swa_metadata.decode_swa_lens
# We treat queries in the same seq as different queries
# and later we only attend by generated indices.
# q arrives pre-padded to self.padded_heads by the outer wrapper.
q = q.unsqueeze(1)
# Prepare SWA cache (num_blocks, swa_block_size, 1, head_bytes)
# Use unsqueeze to preserve strides (handles padded blocks correctly)
swa_cache = self.swa_cache_layer.kv_cache.unsqueeze(-2)
# Reshape KV cache to (num_blocks, block_size, 1, head_bytes)
if kv_cache is not None:
kv_cache = kv_cache.unsqueeze(-2)
# One FlashMLASchedMeta per layer type, shared across all same-type
# layers within this decode step. The first forward call per type
# triggers the in-kernel planner (allocating tile_scheduler_metadata
# and num_splits via PyTorch's graph-aware allocator so CUDA graph
# capture reuses the same addresses on replay); subsequent same-type
# layers see have_initialized=True and skip the planner.
if self.compress_ratio <= 1:
tile_metadata = swa_metadata.tile_sched_swaonly
elif self.compress_ratio == 4:
tile_metadata = swa_metadata.tile_sched_c4a
elif self.compress_ratio == 128:
tile_metadata = swa_metadata.tile_sched_c128a
else:
raise ValueError(
f"Unsupported compress_ratio={self.compress_ratio}; "
"expected 1, 4, or 128."
)
assert tile_metadata is not None, (
"swa_metadata missing tile_sched entry for "
f"compress_ratio={self.compress_ratio}; "
"DeepseekSparseSWAMetadataBuilder.build_tile_scheduler did not "
"allocate one for this layer type."
)
out, _ = flash_mla_with_kvcache(
q=q,
k_cache=swa_cache,
block_table=None,
head_dim_v=512,
tile_scheduler_metadata=tile_metadata,
cache_seqlens=None,
is_fp8_kvcache=True,
indices=swa_indices,
topk_length=swa_lens,
softmax_scale=self.scale,
attn_sink=self.attn_sink,
extra_k_cache=kv_cache if not swa_only else None,
extra_indices_in_kvcache=topk_indices,
extra_topk_length=topk_lens,
out=output.unsqueeze(1),
)
def _forward_prefill(
self,
q: torch.Tensor,
positions: torch.Tensor,
compressed_k_cache: torch.Tensor | None, # Only used when compress_ratio > 1
swa_k_cache: torch.Tensor,
output: torch.Tensor,
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
) -> None:
swa_only = attn_metadata is None
num_prefill_tokens = swa_metadata.num_prefill_tokens
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
# Use pre-computed prefill metadata.
seq_lens = swa_metadata.prefill_seq_lens
gather_lens = swa_metadata.prefill_gather_lens
assert seq_lens is not None
assert gather_lens is not None
# Derive prefill-local token offsets from the full query_start_loc_cpu.
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
assert query_start_loc_cpu is not None
assert query_start_loc is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
if not swa_only:
if self.compress_ratio == 4:
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
topk_indices = topk_indices[:num_prefill_tokens]
else:
# C128A: pre-computed during metadata build.
assert attn_metadata is not None
topk_indices = attn_metadata.c128a_prefill_topk_indices
top_k = topk_indices.shape[-1]
else:
# NOTE(woosuk): topk_indices will not be used for SWA-only layers.
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
top_k = 0
chunk_plan = swa_metadata.get_prefill_chunk_plan(
compress_ratio=self.compress_ratio,
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
)
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
workspace_manager = current_workspace_manager()
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
chunk_size = chunk_end - chunk_start
workspace = workspace_manager.get_simultaneous(
((chunk_size, chunk_M, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
kv, combined_indices_out, combined_lens_out = workspace
if not swa_only:
# Gather compressed KV
assert attn_metadata is not None
block_table = attn_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
compressed_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
gather_lens=None,
block_table=block_table[chunk_start:chunk_end],
block_size=attn_metadata.block_size // self.compress_ratio,
offset=0,
)
# Gather SWA KV
swa_block_table = swa_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
swa_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end],
gather_lens=gather_lens[chunk_start:chunk_end],
block_table=swa_block_table[chunk_start:chunk_end],
block_size=swa_metadata.block_size,
offset=chunk_N,
)
# Combine the topk indices and SWA indices for gathered KV cache
query_start = (
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
)
query_end = (
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
)
combined_indices_out = combined_indices_out[: query_end - query_start]
combined_lens_out = combined_lens_out[: query_end - query_start]
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices[query_start:query_end],
query_start_loc[
num_decodes + chunk_start : num_decodes + chunk_end + 1
],
seq_lens[chunk_start:chunk_end],
gather_lens[chunk_start:chunk_end],
self.window_size,
self.compress_ratio,
top_k,
chunk_M,
chunk_N,
out=(combined_indices_out, combined_lens_out),
left_visible=(
swa_metadata.prefill_left_visible[
num_decode_tokens + query_start : num_decode_tokens + query_end
]
if swa_metadata.prefill_left_visible is not None
else None
),
right_visible=(
swa_metadata.prefill_right_visible[
num_decode_tokens + query_start : num_decode_tokens + query_end
]
if swa_metadata.prefill_right_visible is not None
else None
),
max_image_tokens=self.max_image_tokens,
)
flash_mla_sparse_fwd(
q=q[query_start:query_end],
kv=kv.view(-1, 1, q.shape[-1]),
indices=combined_indices.unsqueeze(1),
sm_scale=self.scale,
attn_sink=self.attn_sink,
topk_length=combined_lens,
out=output[query_start:query_end],
)
File diff suppressed because it is too large Load Diff
-557
View File
@@ -1,557 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
pieces that have no analogue in V3/V32:
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
the fused ``eh_proj``);
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
* V4-specific checkpoint weight-name remapping in ``load_weights``.
"""
import typing
from collections.abc import Callable, Iterable
import regex as re
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
hc_head_fused_kernel_tilelang,
mhc_post_tilelang,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.sequence import IntermediateTensors
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which FusedMoEFactory method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
# Other FP8 linear scales (including shared experts) always use
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
topk_indices_buffer: torch.Tensor,
prefix: str,
aux_stream_list: list[torch.cuda.Stream] | None = None,
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
self.rms_norm_eps = config.rms_norm_eps
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
# fusing them the way V3 does with eh_proj.
self.e_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.e_proj",
)
self.h_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.h_proj",
)
self.hc_eps = config.hc_eps
self.hc_mult = config.hc_mult
self.hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
self.mtp_block = DeepseekV4DecoderLayer(
vllm_config,
prefix,
topk_indices_buffer=topk_indices_buffer,
aux_stream_list=aux_stream_list,
)
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
# reshape to (T, hc_mult, D) — the training-time layout — before
# the fused norm pass so both inputs are 3D-friendly.
previous_hidden_states = previous_hidden_states.view(
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
self.enorm.weight.data,
self.hnorm.weight.data,
self.enorm.variance_epsilon,
self.hc_mult,
)
if self.mtp_block.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, inputs_embeds
)
inputs_embeds = sp_shard(inputs_embeds)
previous_hidden_states = sp_shard(previous_hidden_states)
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
inputs_embeds
).unsqueeze(-2)
mega_gate_metadata = None
if self.use_native_mega_moe:
routing_input_ids = input_ids
if self.mtp_block.use_sequence_parallel:
routing_input_ids = sp_shard(routing_input_ids)
mega_gate_metadata = prepare_mega_gate_routing_metadata(
routing_input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
hidden_states, residual, post_mix, res_mix = self.mtp_block(
positions=positions,
x=hidden_states,
input_ids=input_ids,
mega_gate_metadata=mega_gate_metadata,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.mtp_block.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
# Return the flat pre-hc_head residual so it can be re-fed as the
# next spec step's `previous_hidden_states` when
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
return hidden_states.flatten(1)
class DeepSeekV4MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
topk_tokens = config.index_topk
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
topk_tokens,
dtype=torch.int32,
)
# Three aux streams shared across all MTP layers, mirroring DeepseekV4Model.
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
# to map the exact layer index from weights
self.layers = torch.nn.ModuleDict(
{
str(idx): DeepSeekV4MultiTokenPredictorLayer(
vllm_config,
self.topk_indices_buffer,
f"{prefix}.layers.{idx}",
aux_stream_list=aux_stream_list,
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
# hc_head here so logits are computed from the dense hidden state.
hidden_states = hidden_states.view(
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
)
hidden_states = hc_head_fused_kernel_tilelang(
hidden_states,
mtp_layer.hc_head_fn,
mtp_layer.hc_head_scale,
mtp_layer.hc_head_base,
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
)
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
return logits
class DeepSeekV4MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
self.pad_shared_expert = getattr(
self.quant_config, "weight_block_size", None
) is not None and not _use_sequence_parallel(vllm_config)
self.model = DeepSeekV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def _to_spec_layer_name(self, name: str) -> str:
"""Remap V4 `mtp.{i}.*` names to `model.layers.{num_hidden_layers + i}.*`
so that get_spec_layer_idx_from_weight_name can identify them."""
mtp_layer_idx = 0
for subname in name.split("."):
try:
# we use the first encountered integer
mtp_layer_idx = int(subname)
break
except ValueError:
continue
return name.replace(
f"mtp.{mtp_layer_idx}.",
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
)
def is_unused_checkpoint_weight(self, name: str) -> bool:
name = self._to_spec_layer_name(name)
return get_spec_layer_idx_from_weight_name(self.config, name) is None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Weight name remapping for checkpoint compatibility.
# Maps checkpoint weight paths to model parameter paths.
WEIGHT_NAME_REMAPPING: dict[str, str] = {
".emb.tok_emb.weight": ".embed_tokens.weight",
".head.weight": ".shared_head.head.weight",
".norm.weight": ".shared_head.norm.weight",
}
def _remap_weight_name(name: str) -> str:
"""Remap checkpoint weight names to model parameter names."""
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
if old_pattern in name:
name = name.replace(old_pattern, new_pattern)
return name
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_native_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
else:
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
)
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
# for the rename below based on the model's expert dtype.
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
for name, loaded_weight in weights:
name = self._to_spec_layer_name(name)
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
name = _remap_weight_name(name)
name = self._rewrite_spec_layer_name(spec_layer, name)
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
continue
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[str(spec_layer)].mtp_block.ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
continue
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
break
else:
if ".experts." in name:
# Reinterpret E8M0 scales as uint8 to preserve raw
# exponent bytes; numeric copy_() would zero them.
# Mirrors the main DeepseekV4 loader.
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for mapping in expert_mapping:
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
loaded_params.add(name_mapped)
break
continue
else:
if name.endswith(".ffn.gate.bias"):
# ``e_score_correction_bias`` lives on the gate
# under a different attribute name.
name = name.replace(
".ffn.gate.bias",
".ffn.gate.e_score_correction_bias",
)
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
loaded_layers: set[int] = set()
for param_name in loaded_params:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
if spec_layer is not None:
loaded_layers.add(spec_layer)
for layer_idx in range(
self.model.mtp_start_layer_idx,
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
):
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
raise ValueError(
f"MTP speculative decoding layer {layer_idx} weights "
f"missing from checkpoint. The checkpoint may have "
f"been quantized without including the MTP layers. "
f"Use a checkpoint that includes MTP layer weights, "
f"or disable speculative decoding."
)
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"h_proj",
"e_proj",
"shared_head",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
]
shared_weight_names = ["embed_tokens"]
spec_layer_weight = False
shared_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
if weight_name in shared_weight_names:
shared_weight = True
break
if not spec_layer_weight:
# treat rest weights as weights for transformer layer block
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
elif shared_weight:
# treat shared weights as top level weights
name = name.replace(f"model.layers.{spec_layer}.", "model.")
return name
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NVIDIA-only (cutedsl/cutlass) kernels for DeepSeek V4.
These modules import ``cutlass``/``cutedsl`` at module top level, so they must
not be imported on non-CUDA platforms. Callers should gate on
``vllm.utils.import_utils.has_cutedsl()`` before importing from here.
This ``__init__`` deliberately imports nothing: re-exporting the cutedsl
modules here would eagerly ``import cutlass`` (initializing the CUDA driver) for
anyone who imports ``vllm.models.deepseek_v4``, breaking forked subprocesses.
Import the leaf modules directly under a ``has_cutedsl()``/``is_cuda()`` gate.
"""
@@ -1,425 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Int32, Uint8, Uint32
from cutlass.cute.nvgpu import cpasync
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import _bf16x2_mul, cvt
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
class DequantGatherKCacheKernel(
VllmCuTeDSLJitKernel["DequantGatherKCacheKernel.CompileKey"]
):
# DSv4 geometry. The record format (how much of the row is quantized and
# how wide a scale group is) comes from the compile key: V4 quantizes the
# 448 NoPE dims in groups of 64 and keeps a bf16 RoPE tail, V4.1 quantizes
# all 512 dims in groups of 32 (MXFP8) and has no tail.
head_dim = 512
num_warps = 4
tb_size = num_warps * 32
num_stages = 4
@dataclass(frozen=True)
class CompileKey:
block_size: int
has_gather_lens: bool
fp8_dim: int = 448
group_size: int = 64 # dims per scale
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
head_dim = DequantGatherKCacheKernel.head_dim
fp8_dim = compile_key.fp8_dim
group_size = compile_key.group_size
num_warps = DequantGatherKCacheKernel.num_warps
tb_size = DequantGatherKCacheKernel.tb_size
num_stages = DequantGatherKCacheKernel.num_stages
bf16_dim = head_dim - fp8_dim
data_dim = fp8_dim + bf16_dim * 2
scale_bytes = head_dim // group_size
block_size = compile_key.block_size
# V4.1 stores each token's scales right after its data; V4 puts all of
# a block's data rows ahead of its scale rows.
interleaved = bf16_dim == 0
data_stride = data_dim + scale_bytes if interleaved else data_dim
scale_offset = data_dim if interleaved else block_size * data_dim
scale_stride = data_stride if interleaved else scale_bytes
@cute.jit
def load_g2s(
k_data_slice: cute.Tensor,
k_scale: cute.Tensor,
block_table: cute.Tensor,
s_kdata_slice: cute.Tensor,
s_kscale: cute.Tensor,
req_id,
pos,
lane_id,
stage_id,
):
op = cpasync.CopyG2SOp(cute.nvgpu.LoadCacheMode.GLOBAL)
cp16_atom = cute.make_copy_atom(op, Uint32, num_bits_per_copy=128)
cp_scale_atom = cute.make_copy_atom(
cpasync.CopyG2SOp(), Uint8, num_bits_per_copy=scale_bytes * 8
)
page_id = block_table[req_id, pos // block_size]
block_offset = pos % block_size
# Load the first 512 bytes (32x16B).
idx = lane_id
src = k_data_slice[page_id, block_offset, (None, idx)]
cute.copy(
cp16_atom,
cute.recast_tensor(src, Uint32),
s_kdata_slice[(None, idx), stage_id],
)
# Load the tail 64 bytes.
idx += 32
if idx < cutlass.const_expr(data_dim // 16):
src = k_data_slice[page_id, block_offset, (None, idx)]
cute.copy(
cp16_atom,
cute.recast_tensor(src, Uint32),
s_kdata_slice[(None, idx), stage_id],
)
elif idx == cutlass.const_expr(data_dim // 16):
cute.copy(
cp_scale_atom,
k_scale[page_id, block_offset, None],
s_kscale[None, stage_id],
)
@cute.kernel
def device_kernel(
out: cute.Tensor,
k_data: cute.Tensor,
k_scale: cute.Tensor,
seq_lens: cute.Tensor,
gather_lens: cute.Tensor | None,
block_table: cute.Tensor,
offset: Int32,
):
req_id, worker_id, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
_, num_workers, _ = cute.arch.grid_dim()
# Prepare smem.
smem = cutlass.utils.SmemAllocator()
s_kdata = smem.allocate_tensor(
Uint32,
cute.make_layout((data_dim // 4, num_warps, num_stages)),
byte_alignment=16,
)[None, warp_id, None]
s_kscale = smem.allocate_tensor(
Uint8,
cute.make_layout((scale_bytes, num_warps, num_stages)),
byte_alignment=scale_bytes,
)[None, warp_id, None]
# Prepare for 16B cp.async, also for BF16 smem loads later.
k_data_slice = cute.logical_divide(k_data, (None, None, 16))
s_kdata_16B_slice = cute.logical_divide(s_kdata, (4, None))
# Load FP8 elems in 8B units, so once dequantized, they are 16B units.
s_kdata_8B_slice = cute.logical_divide(s_kdata, (2, None))
# 16B st.global.
out_slice = cute.logical_divide(out, (None, None, 8))
cp_op = cute.nvgpu.CopyUniversalOp()
cp8_atom = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
cp16_atom = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
seq_len = seq_lens[req_id]
gather_len = seq_len
if cutlass.const_expr(gather_lens is not None):
gather_len = gather_lens[req_id] # type: ignore[index]
start_pos = seq_len - gather_len
# Start prefetch.
for i in cutlass.range_constexpr(num_stages - 1):
next_pos = (
start_pos
+ worker_id * num_warps
+ warp_id
+ i * num_workers * num_warps
)
if next_pos < seq_len:
load_g2s(
k_data_slice,
k_scale,
block_table,
s_kdata_16B_slice,
s_kscale,
req_id,
next_pos,
lane_id,
i,
)
cute.arch.cp_async_commit_group()
prefetch_stage = num_stages - 1
compute_stage = 0
# Main loop.
for i in range(
worker_id * num_warps + warp_id,
gather_len,
num_workers * num_warps,
):
pos = start_pos + i
# Prefetch next stage.
next_pos = pos + num_workers * num_warps * (num_stages - 1)
if next_pos < seq_len:
load_g2s(
k_data_slice,
k_scale,
block_table,
s_kdata_16B_slice,
s_kscale,
req_id,
next_pos,
lane_id,
prefetch_stage,
)
prefetch_stage = (prefetch_stage + 1) % num_stages
cute.arch.cp_async_commit_group()
# Wait for gmem->smem to finish.
cute.arch.cp_async_wait_group(num_stages - 1)
cute.arch.sync_warp()
# There are 512 elems per token. As a warp, data0 holds the first
# 256 elems and data1 holds the second 256 elems, i.e. each thread
# holds 8 FP8 elems. This keeps the dequantized 8 BF16 elems as
# contiguous 16B global stores. On Blackwell, this might not be
# necessary as we have 32B global stores, but doing it this way
# does not seem to be slower.
data0 = cute.make_rmem_tensor((2,), Uint32)
data1 = cute.make_rmem_tensor((2,), Uint32)
cute.copy(
cp8_atom, s_kdata_8B_slice[(None, lane_id), compute_stage], data0
)
cute.copy(
cp8_atom,
s_kdata_8B_slice[(None, lane_id + 32), compute_stage],
data1,
)
# Convert to bf16x2 via bit manipulation. FP8 scales are per 64
# elements. An 8-element chunk advances the scale index by
# chunk_id * 8 // group_size.
scale0_u32 = Uint32(s_kscale[lane_id * 8 // group_size, compute_stage])
scale0_bf16x2 = (scale0_u32 << Uint32(23)) | (scale0_u32 << Uint32(7))
scale1_u32 = Uint32(
s_kscale[(lane_id + 32) * 8 // group_size, compute_stage]
)
scale1_bf16x2 = (scale1_u32 << Uint32(23)) | (scale1_u32 << Uint32(7))
# cvt.rn.scaled::n2::ue8m0.bf16x2.e4m3x2 requires PTX 9.2
# (CUDA 13.2).
dequant0 = cute.make_rmem_tensor(4, Uint32)
dequant1 = cute.make_rmem_tensor(4, Uint32)
for j in cutlass.range_constexpr(2):
tmp0 = cvt.fp8x4_to_bf16x4(data0[j])
tmp1 = cvt.fp8x4_to_bf16x4(data1[j])
# BF16 multiply is safe because the scales are exact powers of 2.
dequant0[j * 2] = _bf16x2_mul(tmp0[0], scale0_bf16x2)
dequant1[j * 2] = _bf16x2_mul(tmp1[0], scale1_bf16x2)
dequant0[j * 2 + 1] = _bf16x2_mul(tmp0[1], scale0_bf16x2)
dequant1[j * 2 + 1] = _bf16x2_mul(tmp1[1], scale1_bf16x2)
# Last 64 elems are BF16 tail, corresponds to dequant1 of last
# 8 threads. We have 448 FP8 + 64 BF16 -> 28x 16B for FP8 +
# 8x 16B for BF16. The V4.1 record is fully quantized and has
# no tail.
if cutlass.const_expr(bf16_dim > 0): # noqa: SIM102
if lane_id + 32 >= fp8_dim // 8:
idx = fp8_dim // 16 + (lane_id + 32) - fp8_dim // 8
cute.copy(
cp16_atom,
s_kdata_16B_slice[(None, idx), compute_stage],
dequant1,
)
# Store two 16B BF16 chunks per lane: first half, then second half.
dst = out_slice[req_id, offset + i, (None, lane_id)]
cute.copy(cp16_atom, dequant0, cute.recast_tensor(dst, Uint32))
dst = out_slice[req_id, offset + i, (None, lane_id + 32)]
cute.copy(cp16_atom, dequant1, cute.recast_tensor(dst, Uint32))
compute_stage = (compute_stage + 1) % num_stages
@cute.jit
def host_entrypoint(
out: cute.Tensor,
k_cache: cute.Tensor,
seq_lens: cute.Tensor,
gather_lens: cute.Tensor | None,
block_table: cute.Tensor,
offset: Int32,
stream: CUstream,
):
# Split k_cache into k_data [block_size, fp8_dim + bf16_dim * 2] and
# k_scale [block_size, scale_bytes] views of each block.
k_data = cute.make_tensor(
k_cache.iterator,
layout=cute.make_layout(
(k_cache.shape[0], block_size, data_dim),
stride=(k_cache.stride[0], data_stride, 1),
),
)
# k_data_slice: [num_blocks, block_size, (16, data_dim/16)]
# s_kdata_slice: [(4, data_dim/16), num_stages]
k_scale = cute.make_tensor(
k_cache.iterator + scale_offset,
layout=cute.make_layout(
(k_cache.shape[0], block_size, scale_bytes),
stride=(k_cache.stride[0], scale_stride, 1),
),
)
num_reqs = out.shape[0]
max_rows = out.shape[1] - offset
num_workers = cutlass.max(
1,
cutlass.min(cute.ceil_div(max_rows, 4), cute.ceil_div(8192, num_reqs)),
)
grid = (num_reqs, num_workers, 1)
device_kernel(
out,
k_data,
k_scale,
seq_lens,
gather_lens,
block_table,
offset,
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
*,
block_size: int,
has_gather_lens: bool,
fp8_dim: int = 448,
group_size: int = 64,
) -> CompileKey:
return self.CompileKey(
block_size=block_size,
has_gather_lens=has_gather_lens,
fp8_dim=fp8_dim,
group_size=group_size,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
block_size = vllm_config.cache_config.block_size
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
return self._trace_dispatch(self.dispatch)(
zip_inputs(
dict(
block_size=block_size,
has_gather_lens=True,
enabled=True,
),
dict(
block_size=max(1, block_size // 4),
has_gather_lens=False,
enabled=4 in compress_ratios,
),
dict(
block_size=max(1, block_size // 128),
has_gather_lens=False,
enabled=128 in compress_ratios,
),
),
valid_block_size=block_size > 0,
_when=lambda *, enabled, valid_block_size: enabled and valid_block_size,
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_reqs = cute.sym_int()
head_dim = self.head_dim
fp8_dim = compile_key.fp8_dim
block_size = compile_key.block_size
head_bytes = (
fp8_dim + (head_dim - fp8_dim) * 2 + head_dim // compile_key.group_size
)
out = make_fake_tensor(BFloat16, (num_reqs, cute.sym_int(), head_dim), 16)
k_cache = cute.runtime.make_fake_tensor(
Uint8,
(cute.sym_int(), block_size, head_bytes),
stride=(cute.sym_int64(divisibility=32), head_bytes, 1),
assumed_align=32,
)
seq_lens = make_fake_tensor(Int32, (num_reqs,))
gather_lens = (
make_fake_tensor(Int32, (num_reqs,))
if compile_key.has_gather_lens
else None
)
block_table = make_fake_tensor(Int32, (num_reqs, cute.sym_int()))
return (
out,
k_cache,
seq_lens,
gather_lens,
block_table,
Int32(0),
)
@kernel_launcher
def __call__(
self,
*,
out: torch.Tensor,
k_cache: torch.Tensor,
seq_lens: torch.Tensor,
gather_lens: torch.Tensor | None,
block_table: torch.Tensor,
block_size: int,
offset: int,
fp8_dim: int = 448,
group_size: int = 64,
) -> CuTeDSLLaunchSpec["DequantGatherKCacheKernel.CompileKey"]:
compile_key = self.dispatch(
block_size=block_size,
has_gather_lens=gather_lens is not None,
fp8_dim=fp8_dim,
group_size=group_size,
)
launch_args = (
out,
k_cache,
seq_lens,
gather_lens,
block_table,
offset,
)
return compile_key, launch_args
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL = DequantGatherKCacheKernel()
@@ -1,708 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int64, Uint8, Uint32, const_expr
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
_bf16x2_abs,
_bf16x2_max,
cvt,
recast_val,
torch_to_cute_dtype,
)
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.vllm_flash_attn.cute import utils as cute_utils
# MXFP4: 32 elements per block, packed 2 nibbles per byte, ue8m0 block scale.
MXFP4_BLOCK_SIZE = 32
@cute.jit
def _load_q_and_rope(
positions: cute.Tensor,
q: cute.Tensor,
cos_sin_cache: cute.Tensor,
num_heads: cutlass.Constexpr,
coarsen: cutlass.Constexpr,
tb_size: cutlass.Constexpr,
subwarp_size: cutlass.Constexpr,
threads_per_token: cutlass.Constexpr,
nope_dim: cutlass.Constexpr,
rope_dim: cutlass.Constexpr,
cos_sin_dtype: cutlass.Constexpr,
):
"""Compute thread indices, load Q (BF16), and apply interleaved RoPE.
Returns a tuple
(q_bf16x2, tid, global_tid, sublane, token_id, head_tile_id,
head_start, in_bounds, num_token_heads)
where ``q_bf16x2`` is a (coarsen, 8) rmem tile of Uint32 packed
bf16x2 pairs covering the 16 BF16 lanes owned by this thread for
each of ``coarsen`` heads. RoPE is applied in place to the
trailing ``rope_dim`` lanes; the leading nope lanes pass through.
"""
block_id, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
num_tokens = q.shape[0]
num_token_heads = num_tokens * num_heads
global_tid = block_id * tb_size + tid
global_subwarp_id = global_tid // subwarp_size
sublane = tid % subwarp_size
token_id = global_subwarp_id // (num_heads // coarsen)
head_tile_id = global_subwarp_id % (num_heads // coarsen)
head_start = head_tile_id * coarsen
# NOTE: token_id may exceed bounds, hence we need to add load/store guards
# we can't do early exit because CuteDSL doesn't support it. and we also need
# all threads in a warp to be active since we utilize warp shuffle later.
# must_in_bounds is constexpr, True when a token spans a whole number of
# threadblocks. Then num_tokens * threads_per_token is always a multiple of
# tb_size, so the trailing block never straddles num_tokens and the compiler
# removes the bounds check. Otherwise the guard has to stay: e.g. 32 heads
# with coarsen=4 needs 64 threads per token, so an odd num_tokens leaves half
# of the last block addressing token num_tokens.
must_in_bounds = cutlass.const_expr(threads_per_token % tb_size == 0)
in_bounds = must_in_bounds or (token_id < num_tokens)
cp_op = cute.nvgpu.CopyUniversalOp()
_layout = cute.make_layout((coarsen, 8), stride=(8, 1))
q_bf16x2 = cute.make_rmem_tensor(_layout, Uint32)
if in_bounds:
# we can't do cute.copy() on the whole 2D tile directly because
# cute.copy() wants the 1st mode to be covered by the copy atom,
# and other modes as for loop. there is no fast way to
# "transpose" the tensor view.
q_tile = cute.local_tile(
q[token_id, None, None],
tiler=(coarsen, 16),
coord=(head_tile_id, sublane),
)
cp_u32x8 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
for i in cutlass.range_constexpr(coarsen):
src = cute.recast_tensor(q_tile[i, None], Uint32)
cute.copy(cp_u32x8, src, q_bf16x2[i, None])
# RoPE applies only to the trailing rope_dim values. We keep the rounded
# BF16 result in q_bits so the later amax and quantization see BF16.
# cos_sin_cache layout: [max_pos, rope_dim]
if in_bounds and sublane * 16 >= nope_dim:
cos_vals = cute.make_rmem_tensor((8,), Float32)
sin_vals = cute.make_rmem_tensor((8,), Float32)
pos = positions[token_id]
# select 8 elems from cos and sin
cos_id = sublane - nope_dim // 16
sin_id = cos_id + rope_dim // 16
cos_src = cute.local_tile(cos_sin_cache[pos, None], tiler=(8,), coord=(cos_id,))
sin_src = cute.local_tile(cos_sin_cache[pos, None], tiler=(8,), coord=(sin_id,))
cp_f32x8 = cute.make_copy_atom(cp_op, Float32, num_bits_per_copy=256)
cp_u32x4 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
if const_expr(cos_sin_dtype is Float32):
cute.copy(cp_f32x8, cos_src, cos_vals)
cute.copy(cp_f32x8, sin_src, sin_vals)
else:
cos_bf16x2 = cute.make_rmem_tensor((4,), Uint32)
sin_bf16x2 = cute.make_rmem_tensor((4,), Uint32)
cute.copy(cp_u32x4, cute.recast_tensor(cos_src, Uint32), cos_bf16x2)
cute.copy(cp_u32x4, cute.recast_tensor(sin_src, Uint32), sin_bf16x2)
for i in cutlass.range_constexpr(4):
cos0, cos1 = cvt.bf16x2_to_fp32x2(cos_bf16x2[i])
sin0, sin1 = cvt.bf16x2_to_fp32x2(sin_bf16x2[i])
cos_vals[i * 2] = cos0
cos_vals[i * 2 + 1] = cos1
sin_vals[i * 2] = sin0
sin_vals[i * 2 + 1] = sin1
for i in cutlass.range_constexpr(coarsen):
for j in cutlass.range_constexpr(8):
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j])
rot0 = q0 * cos_vals[j] - q1 * sin_vals[j]
rot1 = q0 * sin_vals[j] + q1 * cos_vals[j]
# convert back to BF16 to match numerics
q_bf16x2[i, j] = cvt.fp32x2_to_bf16x2(rot0, rot1)
return (
q_bf16x2,
tid,
global_tid,
sublane,
token_id,
head_tile_id,
head_start,
in_bounds,
num_token_heads,
)
class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]):
tb_size = 128
@dataclass(frozen=True)
class CompileKey:
head_dim: int
rope_dim: int
num_heads: int
cos_sin_dtype: type[cutlass.Numeric]
weights_out_dtype: type[cutlass.Numeric]
coarsen: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
head_dim = compile_key.head_dim
rope_dim = compile_key.rope_dim
nope_dim = head_dim - rope_dim
num_heads = compile_key.num_heads
cos_sin_dtype = compile_key.cos_sin_dtype
# process multiple heads at the same time to armotize RoPE load costs
coarsen = compile_key.coarsen
tb_size = IndexerQMxFp4Kernel.tb_size
weights_out_dtype = compile_key.weights_out_dtype
assert num_heads % coarsen == 0
# later we will use 32B load = 16 BF16 elems
# thus, head_dim=128 requires 8 threads to handle.
# let's call subwarp = 8 threads.
subwarp_size = head_dim // 16
threads_per_token = (num_heads // coarsen) * subwarp_size
@cute.kernel
def device_kernel(
positions: cute.Tensor,
q: cute.Tensor,
cos_sin_cache: cute.Tensor,
weights: cute.Tensor,
q_quant: cute.Tensor,
q_scale: cute.Tensor,
weights_out: cute.Tensor,
scale: Float32,
):
(
q_bf16x2,
tid,
global_tid,
sublane,
token_id,
head_tile_id,
head_start,
in_bounds,
num_token_heads,
) = _load_q_and_rope(
positions,
q,
cos_sin_cache,
num_heads,
coarsen,
tb_size,
subwarp_size,
threads_per_token,
nope_dim,
rope_dim,
cos_sin_dtype,
)
cp_op = cute.nvgpu.CopyUniversalOp()
# layout: [coarsen, 8]
q_fp4_tile = cute.local_tile(
q_quant[token_id, None, None],
tiler=(coarsen, 8),
coord=(head_tile_id, sublane),
)
for i in cutlass.range_constexpr(coarsen):
# compute amax in packed bf16x2 to save instructions
# Each thread holds 16 elems. Two adjacent threads form one 32-elem
# MXFP4 block, so a width-2 shuffle gives the block amax.
amax_bf16x2 = _bf16x2_abs(q_bf16x2[i, 0])
for j in cutlass.range_constexpr(1, 8):
amax_bf16x2 = _bf16x2_max(amax_bf16x2, _bf16x2_abs(q_bf16x2[i, j]))
amax_bf16x2 = cute_utils.warp_reduce(
amax_bf16x2,
_bf16x2_max,
width=MXFP4_BLOCK_SIZE // 16,
)
amax_pair = cvt.bf16x2_to_fp32x2(amax_bf16x2)
amax = cute_utils.fmax(amax_pair[0], amax_pair[1])
if in_bounds:
# compute block scale with bit manipulation
# UE8M0 stores ceil(log2(fp4_scale)) + 127. Adding the mantissa mask
# increments the exponent whenever fp4_scale is not exactly a power of 2
eps = cutlass.const_expr(float.fromhex("0x6p-126"))
fp4_scale = cute_utils.fmax(amax, eps) * Float32(1.0 / 6.0)
bits = recast_val(fp4_scale, Uint32)
ue8m0 = cute_utils.shr_u32(
bits + Uint32(0x7FFFFF), Uint32(23)
) & Uint32(0xFF)
# Only one of the two threads in an MXFP4 block writes the shared scale.
if tid % 2 == 0:
mx_block = sublane // 2
q_scale[token_id, head_start + i, mx_block] = Uint8(ue8m0)
# If scale = 2^A and ue8m0 = A + 127, then inverse scale has exponent
# -A + 127 = 254 - ue8m0.
inv_scale_bits = (Uint32(254) - ue8m0) << Uint32(23)
inv_fp4_scale = recast_val(inv_scale_bits, Float32)
vals = cute.make_rmem_tensor(16, Float32)
for j in cutlass.range_constexpr(8):
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j])
vals[j * 2] = q0 * inv_fp4_scale
vals[j * 2 + 1] = q1 * inv_fp4_scale
# pack to FP4
packed = cute.make_rmem_tensor((2,), Uint32)
packed[0] = cvt.fp32x8_to_fp4x8(vals, 0)
packed[1] = cvt.fp32x8_to_fp4x8(vals, 8)
dst = q_fp4_tile[i, None]
cp_u32x2 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
cute.copy(cp_u32x2, packed, cute.recast_tensor(dst, Uint32))
# Weight scaling is independent of the Q subwarp work. The first
# num_tokens * num_heads logical threads cover one weight each.
if global_tid < num_token_heads:
weight_token_id = global_tid // num_heads
weight_head_id = global_tid % num_heads
weights_out[weight_token_id, weight_head_id] = (
weights[weight_token_id, weight_head_id].to(Float32) * scale
).to(weights_out_dtype)
@cute.jit
def host_entrypoint(
positions: cute.Tensor,
q: cute.Tensor,
cos_sin_cache: cute.Tensor,
weights: cute.Tensor,
q_quant: cute.Tensor,
q_scale: cute.Tensor,
weights_out: cute.Tensor,
scale: Float32,
stream: CUstream,
):
total_threads = q.shape[0] * threads_per_token
grid = (cute.ceil_div(total_threads, tb_size), 1, 1)
device_kernel(
positions,
q,
cos_sin_cache,
weights,
q_quant,
q_scale,
weights_out,
scale,
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
*,
cos_sin_dtype: type[cutlass.Numeric],
weights_out_dtype: type[cutlass.Numeric],
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
cos_sin_dtype=cos_sin_dtype,
weights_out_dtype=weights_out_dtype,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
from vllm.v1.attention.backends.mla.indexer import dsa_indexer_uses_fp4
hf_config = vllm_config.model_config.hf_config
if hf_config is None or not dsa_indexer_uses_fp4(vllm_config):
return []
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
# The DeepSeek V4.1 sparse-logits indexer takes bf16 weights; the
# dense scoring kernels take fp32.
weights_out_dtypes: tuple[type[cutlass.Numeric], ...] = (Float32,)
if vllm_config.attention_config.indexer_sparse_logits:
weights_out_dtypes = (Float32, BFloat16)
return self._trace_dispatch(self.dispatch)(
head_dim=head_dim,
rope_dim=rope_dim,
num_heads=num_heads,
cos_sin_dtype=Float32,
weights_out_dtype=weights_out_dtypes,
coarsen=(1, 4),
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_tokens = cute.sym_int()
max_pos = cute.sym_int()
q = make_fake_tensor(
BFloat16,
(num_tokens, compile_key.num_heads, compile_key.head_dim),
divisibility=16,
)
positions = make_fake_tensor(Int64, (num_tokens,), divisibility=1)
cos_sin_cache = make_fake_tensor(
compile_key.cos_sin_dtype,
(max_pos, compile_key.rope_dim),
divisibility=8,
)
weights = make_fake_tensor(
BFloat16, (num_tokens, compile_key.num_heads), divisibility=8
)
q_packed = make_fake_tensor(
Uint8,
(num_tokens, compile_key.num_heads, compile_key.head_dim // 2),
divisibility=16,
)
q_scale = make_fake_tensor(
Uint8,
(
num_tokens,
compile_key.num_heads,
compile_key.head_dim // MXFP4_BLOCK_SIZE,
),
divisibility=4,
)
weights_out = make_fake_tensor(
compile_key.weights_out_dtype,
(num_tokens, compile_key.num_heads),
divisibility=4,
)
return (
positions,
q,
cos_sin_cache,
weights,
q_packed,
q_scale,
weights_out,
Float32(0.0),
)
@kernel_launcher
def __call__(
self,
*,
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
weights_softmax_scale: float,
weights_head_scale: float,
q_packed: torch.Tensor,
q_scale: torch.Tensor,
weights_out: torch.Tensor,
) -> CuTeDSLLaunchSpec["IndexerQMxFp4Kernel.CompileKey"]:
num_tokens, num_heads, head_dim = q.shape
compile_key = self.dispatch(
head_dim=head_dim,
rope_dim=cos_sin_cache.shape[-1],
num_heads=num_heads,
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
weights_out_dtype=torch_to_cute_dtype(weights_out.dtype),
coarsen=1 if num_tokens < 512 else 4,
)
launch_args = (
positions,
q,
cos_sin_cache,
weights,
q_packed,
q_scale,
weights_out,
float(weights_softmax_scale * weights_head_scale),
)
return compile_key, launch_args
class IndexerQFp8Kernel(VllmCuTeDSLJitKernel["IndexerQFp8Kernel.CompileKey"]):
tb_size = 128
@dataclass(frozen=True)
class CompileKey:
head_dim: int
rope_dim: int
num_heads: int
cos_sin_dtype: type[cutlass.Numeric]
coarsen: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
head_dim = compile_key.head_dim
rope_dim = compile_key.rope_dim
nope_dim = head_dim - rope_dim
num_heads = compile_key.num_heads
cos_sin_dtype = compile_key.cos_sin_dtype
# process multiple heads at the same time to armotize RoPE load costs
coarsen = compile_key.coarsen
tb_size = IndexerQFp8Kernel.tb_size
assert num_heads % coarsen == 0
# later we will use 32B load = 16 BF16 elems
# thus, head_dim=128 requires 8 threads to handle.
# let's call subwarp = 8 threads.
subwarp_size = head_dim // 16
threads_per_token = (num_heads // coarsen) * subwarp_size
# Each subwarp owns `coarsen` heads; we use the first `coarsen`
# threads of the subwarp to write the per-head weights using the
# fp8 scale computed in the matching loop iteration.
assert coarsen <= subwarp_size, (
f"FP8 kernel requires coarsen ({coarsen}) <= "
f"subwarp_size ({subwarp_size}) for the weight-fold step"
)
@cute.kernel
def device_kernel(
positions: cute.Tensor,
q: cute.Tensor,
cos_sin_cache: cute.Tensor,
weights: cute.Tensor,
q_fp8: cute.Tensor,
weights_out: cute.Tensor,
scale: Float32,
):
(
q_bf16x2,
_tid,
_global_tid,
sublane,
token_id,
head_tile_id,
head_start,
in_bounds,
_num_token_heads,
) = _load_q_and_rope(
positions,
q,
cos_sin_cache,
num_heads,
coarsen,
tb_size,
subwarp_size,
threads_per_token,
nope_dim,
rope_dim,
cos_sin_dtype,
)
cp_op = cute.nvgpu.CopyUniversalOp()
# layout: [coarsen, 16] bytes (one e4m3fn per element).
q_fp8_tile = cute.local_tile(
q_fp8[token_id, None, None],
tiler=(coarsen, 16),
coord=(head_tile_id, sublane),
)
for i in cutlass.range_constexpr(coarsen):
# Reduce amax across the full head_dim: each thread already holds
# the max over its 16 lanes; a width=subwarp_size warp shuffle
# spreads the head-wide max to every lane in the subwarp.
amax_bf16x2 = _bf16x2_abs(q_bf16x2[i, 0])
for j in cutlass.range_constexpr(1, 8):
amax_bf16x2 = _bf16x2_max(amax_bf16x2, _bf16x2_abs(q_bf16x2[i, j]))
amax_bf16x2 = cute_utils.warp_reduce(
amax_bf16x2,
_bf16x2_max,
width=subwarp_size,
)
amax_pair = cvt.bf16x2_to_fp32x2(amax_bf16x2)
amax = cute_utils.fmax(amax_pair[0], amax_pair[1])
# scale = max(amax, eps) / fp8_max, then rounded UP to the next
# power of two. Adding the mantissa mask before shifting out the
# mantissa bumps the exponent whenever s isn't a pure pow2.
fp32_scale = cute_utils.fmax(amax, Float32(1e-4)) * Float32(1.0 / 448.0)
bits = recast_val(fp32_scale, Uint32)
scale_exp = cute_utils.shr_u32(
bits + Uint32(0x7FFFFF), Uint32(23)
) & Uint32(0xFF)
# rounded scale = 2^(scale_exp - 127); bit pattern is scale_exp << 23
fp8_scale_bits = scale_exp << Uint32(23)
fp8_scale = recast_val(fp8_scale_bits, Float32)
# inverse = 2^-(scale_exp - 127); bit pattern is (254 - scale_exp) << 23
inv_scale_bits = (Uint32(254) - scale_exp) << Uint32(23)
inv_fp8_scale = recast_val(inv_scale_bits, Float32)
# Weight fold: weights_out = weights * q_scale * scale_combined.
# All threads in the subwarp share the same fp8_scale after the
# warp_reduce above, so we let thread `sublane == i` write the
# weight for head `head_start + i`.
if in_bounds and sublane == i:
head_id = head_start + i
weights_out[token_id, head_id] = (
weights[token_id, head_id].to(Float32) * scale * fp8_scale
)
if in_bounds:
# 16 BF16 → 16 e4m3 bytes per thread, packed into 4 b32s
# (one cp.async-shaped 128-bit store per row).
packed = cute.make_rmem_tensor((4,), Uint32)
for j in cutlass.range_constexpr(4):
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j * 2])
q2, q3 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j * 2 + 1])
packed[j] = cvt.fp32x4_to_fp8x4(
q0 * inv_fp8_scale,
q1 * inv_fp8_scale,
q2 * inv_fp8_scale,
q3 * inv_fp8_scale,
)
dst = q_fp8_tile[i, None]
cp_u32x4 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cute.copy(cp_u32x4, packed, cute.recast_tensor(dst, Uint32))
@cute.jit
def host_entrypoint(
positions: cute.Tensor,
q: cute.Tensor,
cos_sin_cache: cute.Tensor,
weights: cute.Tensor,
q_fp8: cute.Tensor,
weights_out: cute.Tensor,
scale: Float32,
stream: CUstream,
):
total_threads = q.shape[0] * threads_per_token
grid = (cute.ceil_div(total_threads, tb_size), 1, 1)
device_kernel(
positions,
q,
cos_sin_cache,
weights,
q_fp8,
weights_out,
scale,
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
*,
cos_sin_dtype: type[cutlass.Numeric],
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
cos_sin_dtype=cos_sin_dtype,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
from vllm.v1.attention.backends.mla.indexer import dsa_indexer_uses_fp4
hf_config = vllm_config.model_config.hf_config
if hf_config is None or dsa_indexer_uses_fp4(vllm_config):
return []
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
return self._trace_dispatch(self.dispatch)(
head_dim=head_dim,
rope_dim=rope_dim,
num_heads=num_heads,
cos_sin_dtype=Float32,
coarsen=(1, 4),
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_tokens = cute.sym_int()
max_pos = cute.sym_int()
q = make_fake_tensor(
BFloat16,
(num_tokens, compile_key.num_heads, compile_key.head_dim),
divisibility=16,
)
positions = make_fake_tensor(Int64, (num_tokens,), divisibility=1)
cos_sin_cache = make_fake_tensor(
compile_key.cos_sin_dtype,
(max_pos, compile_key.rope_dim),
divisibility=8,
)
weights = make_fake_tensor(
BFloat16, (num_tokens, compile_key.num_heads), divisibility=8
)
q_fp8 = make_fake_tensor(
Uint8,
(num_tokens, compile_key.num_heads, compile_key.head_dim),
divisibility=16,
)
weights_out = make_fake_tensor(
Float32, (num_tokens, compile_key.num_heads), divisibility=4
)
return (
positions,
q,
cos_sin_cache,
weights,
q_fp8,
weights_out,
Float32(0.0),
)
@kernel_launcher
def __call__(
self,
*,
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
weights_softmax_scale: float,
weights_head_scale: float,
q_fp8: torch.Tensor,
weights_out: torch.Tensor,
) -> CuTeDSLLaunchSpec["IndexerQFp8Kernel.CompileKey"]:
num_tokens, num_heads, head_dim = q.shape
compile_key = self.dispatch(
head_dim=head_dim,
rope_dim=cos_sin_cache.shape[-1],
num_heads=num_heads,
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
coarsen=1 if num_tokens < 512 else 4,
)
launch_args = (
positions,
q,
cos_sin_cache,
weights,
q_fp8.view(torch.uint8),
weights_out,
float(weights_softmax_scale * weights_head_scale),
)
return compile_key, launch_args
_INDEXER_Q_MXFP4_KERNEL = IndexerQMxFp4Kernel()
_INDEXER_Q_FP8_KERNEL = IndexerQFp8Kernel()
@@ -1,158 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
import torch.nn as nn
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
)
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
def compute_fp8_einsum_recipe(
block_size: int = 128,
) -> tuple[tuple[int, int, int], bool]:
"""fp8_einsum recipe + scale layout for the current GPU arch.
SM90 keeps block-row FP32 scales. SM100 uses packed per-row E8M0 scales.
Returns ``(einsum_recipe, tma_aligned_scales)`` for ``deep_gemm_fp8_o_proj``.
"""
cap = current_platform.get_device_capability()
assert cap is not None, "DeepseekV4 attention requires a CUDA device"
einsum_recipe = (1, 128, 128) if cap.major <= 9 else (1, 1, block_size)
tma_aligned_scales = cap.major >= 10
return einsum_recipe, tma_aligned_scales
def deep_gemm_fp8_o_proj(
o: torch.Tensor | QuantizedActivation,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
wo_a: nn.Module,
wo_b: Callable[[torch.Tensor], torch.Tensor],
*,
n_groups: int,
heads_per_group: int,
nope_dim: int,
rope_dim: int,
o_lora_rank: int,
einsum_recipe: tuple[int, int, int],
tma_aligned_scales: bool,
) -> torch.Tensor:
"""O projection: inverse RoPE + grouped wo_a + wo_b.
Shared by the FlashMLA and FlashInfer CUDA backends. The attention
layer selects the recipe at initialization. ``wo_b`` is any callable over
the flattened ``z``: the projection module itself, or a wrapper that also
reduce-scatters its output (DeepSeek-V4.1 GEMM-RS).
A QuantizedActivation ``o`` was already rotated and cast by the attention
kernel, one ``wo_a`` group per slot with the live groups first, so only
the padding groups are sliced off.
"""
use_fp8 = wo_a.weight.dtype == torch.float8_e4m3fn
if isinstance(o, QuantizedActivation):
assert use_fp8
o_proj_input, o_scale = o.data[:, :n_groups], o.scale[:, :n_groups]
else:
o_proj_input, o_scale = fused_inv_rope_fp8_quant(
o,
positions,
cos_sin_cache,
n_groups=n_groups,
heads_per_group=heads_per_group,
nope_dim=nope_dim,
rope_dim=rope_dim,
quant_group_size=einsum_recipe[2],
tma_aligned_scales=tma_aligned_scales,
quantize=use_fp8,
)
z = torch.empty(
(o_proj_input.shape[0], n_groups, o_lora_rank),
device=o_proj_input.device,
dtype=torch.bfloat16,
)
if use_fp8:
weight_scale = (
wo_a.weight_scale
if hasattr(wo_a, "weight_scale")
else wo_a.weight_scale_inv
)
fp8_einsum(
"bhr,hdr->bhd",
(o_proj_input, o_scale),
(wo_a.weight, weight_scale),
z,
recipe=einsum_recipe,
)
else:
grouped_weight = wo_a.weight.view(n_groups, o_lora_rank, -1)
torch.bmm(
o_proj_input.transpose(0, 1),
grouped_weight.transpose(1, 2),
out=z.transpose(0, 1),
)
return wo_b(z.flatten(1))
# TRTLLM-gen's DSv4 RopeQuant epilogue (flashinfer-ai/flashinfer#4918) has one
# fixed schedule: 128 query heads in groups of 8 512-wide heads, and FP8 with
# one packed UE8M0 scale per 128 elements.
_ROPE_QUANT_NUM_HEADS = 128
_ROPE_QUANT_HEADS_PER_GROUP = 8
_ROPE_QUANT_HEAD_DIM = 512
_ROPE_QUANT_BLOCK = 128
def rope_quant_unsupported_reason(layer: torch.nn.Module) -> str | None:
"""Why ``layer`` cannot take FlashInfer's fused inverse RoPE + FP8 output.
DSv4 always has 8 heads per ``wo_a`` group at any TP size, and the rest of
the kernel's fixed shape (head/RoPE dims, per-128 einsum scales, FP32 RoPE
cache) is what the SM100 layer uses anyway. What varies is the local head
count and the configs the layer also serves unquantized.
"""
if layer.n_local_heads != _ROPE_QUANT_NUM_HEADS:
# Padding fewer heads up to the kernel's fixed 128 costs more
# attention than the fused epilogue saves.
return f"RopeQuant needs {_ROPE_QUANT_NUM_HEADS} local query heads"
if layer.kv_cache_torch_dtype != torch.float8_e4m3fn:
return "RopeQuant needs the per-tensor FP8 KV cache"
if layer.wo_a.weight.dtype != torch.float8_e4m3fn:
return "RopeQuant needs an FP8 wo_a"
return None
def rope_quant_attn_out(num_tokens: int, device: torch.device) -> QuantizedActivation:
"""Allocate the output pair the RopeQuant kernel writes.
Values are group-major, ``[num_tokens, 16, 8 * 512]`` over physical
``[group, token, K]``. Scales are MN-major packed UE8M0, one int32 per
head over its four 128-wide blocks. Both are what ``fp8_einsum`` consumes.
"""
n_groups = _ROPE_QUANT_NUM_HEADS // _ROPE_QUANT_HEADS_PER_GROUP
group_width = _ROPE_QUANT_HEADS_PER_GROUP * _ROPE_QUANT_HEAD_DIM
data = torch.empty(
(n_groups, num_tokens, group_width), dtype=torch.float8_e4m3fn, device=device
).transpose(0, 1)
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
scale = torch.empty(
(n_groups, group_width // (_ROPE_QUANT_BLOCK * 4), aligned),
dtype=torch.int32,
device=device,
).permute(2, 0, 1)[:num_tokens]
return QuantizedActivation(
data=data,
scale=scale,
orig_dtype=torch.bfloat16,
orig_shape=data.shape,
quant_key=kFp8Dynamic128Sym,
)
File diff suppressed because it is too large Load Diff
@@ -1,21 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compatibility imports for the platform-neutral DeepSeek-V4 vision model."""
from ..common.vl_model import (
DeepseekV4ForConditionalGeneration,
)
from ..common.vl_model import (
_make_deepseek_v4_vl_weights_mapper as _make_common_vl_weights_mapper,
)
from .model import _make_deepseek_v4_weights_mapper
def _make_deepseek_v4_vl_weights_mapper(expert_dtype: str, image_enabled: bool):
"""Retain the original NVIDIA helper signature for downstream imports."""
return _make_common_vl_weights_mapper(
_make_deepseek_v4_weights_mapper(expert_dtype), image_enabled
)
__all__ = ["DeepseekV4ForConditionalGeneration"]
-229
View File
@@ -1,229 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Quantization config for DeepSeek V4."""
from __future__ import annotations
from typing import TYPE_CHECKING, cast
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
if TYPE_CHECKING:
from transformers import PreTrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
def align_mtp_quant_config(
quant_config: QuantizationConfig | None, config: PreTrainedConfig
) -> None:
"""Re-key the quant config's MTP entries onto the draft module prefixes.
``_make_deepseek_v4_weights_mapper`` renames the checkpoint's ``mtp.{i}.``
layers to ``model.mtp.{i}.``, but the draft modules are built under
``model.layers.{num_hidden_layers + i}.``. Without this remap no per-layer
entry matches, so every MTP linear falls back to the global scheme. On a
mixed checkpoint (FP8 attention and shared experts, NVFP4 routed experts)
that builds NVFP4 parameters for FP8 weights and the load then fails a
shape assertion.
"""
from vllm.model_executor.models.utils import WeightsMapper
if quant_config is None:
return
quant_config.apply_vllm_mapper(
WeightsMapper(
orig_to_new_prefix={
f"model.mtp.{i}.": f"model.layers.{config.num_hidden_layers + i}."
for i in range(config.num_nextn_predict_layers)
}
)
)
class DeepseekV4FP8Config(Fp8Config):
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
DeepSeek V4 checkpoints always use FP8 block quantization for
linear/attention layers. The MoE expert weights vary by checkpoint:
- ``expert_dtype="fp4"`` (e.g. DeepSeek-V4-Flash): MXFP4 experts
with ue8m0 (e8m0fnu) FP8 linear scales.
- ``expert_dtype="fp8"`` (e.g. DeepSeek-V4-Flash-Base): FP8 block
experts with float32 FP8 linear scales.
The dispatch and the linear scale dtype are both keyed off
``expert_dtype`` from the model's hf_config; missing values default
to ``"fp4"`` so existing FP4 checkpoints stay unchanged.
NOTE: ``expert_dtype`` is resolved lazily because this config is
constructed during VllmConfig setup, before ``set_current_vllm_config``
is active. Reading hf_config eagerly in ``__init__`` would always see
the default ``"fp4"`` and silently misroute Flash-Base checkpoints.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._resolved_expert_dtype: str | None = None
self._resolved_moe_quant_algo: str | None = None
self._nvfp4_config: ModelOptNvFp4Config | None = None
# ``is_scale_e8m0`` is a property that resolves on first read,
# by which time the current vllm_config has been set.
@property
def expert_dtype(self) -> str:
if self._resolved_expert_dtype is None:
try:
hf_config = get_current_vllm_config().model_config.hf_config
except Exception:
# vllm_config not yet set; defer the decision until a
# later call lands inside set_current_vllm_config.
return "fp4"
expert_dtype = getattr(hf_config, "expert_dtype", "fp4")
if expert_dtype not in _DEEPSEEK_V4_EXPERT_DTYPES:
raise ValueError(
f"Unsupported DeepSeek V4 expert_dtype={expert_dtype!r}; "
f"expected one of {_DEEPSEEK_V4_EXPERT_DTYPES}."
)
self._resolved_expert_dtype = expert_dtype
from vllm.logger import init_logger
init_logger(__name__).info_once(
"DeepSeek V4 expert_dtype resolved to %r", expert_dtype
)
return self._resolved_expert_dtype
@property
def is_scale_e8m0(self) -> bool:
# FP4 checkpoints store FP8 linear scales as e8m0fnu; FP8 expert
# checkpoints (Flash-Base) store them as float32.
return self.expert_dtype == "fp4"
def _resolve_moe_overrides(self) -> None:
if self._resolved_moe_quant_algo is not None:
return
try:
hf_config = get_current_vllm_config().model_config.hf_config
except Exception:
return
quant_cfg = getattr(hf_config, "quantization_config", None) or {}
algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None
self._resolved_moe_quant_algo = algo or ""
@property
def moe_quant_algo(self) -> str:
self._resolve_moe_overrides()
return self._resolved_moe_quant_algo or ""
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
if self._nvfp4_config is None:
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
self._nvfp4_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
group_size=16,
quantization_args=(
self.online_quantization_config.args
if self.online_quantization_config is not None
else None
),
)
return self._nvfp4_config
@classmethod
def get_name(cls) -> QuantizationMethods:
return "deepseek_v4_fp8"
@staticmethod
def _is_quark_mxfp4_ocp(hf_quant_cfg: dict) -> bool:
"""True for AMD-Quark exports whose global scheme is MXFP4."""
weight = (hf_quant_cfg.get("global_quant_config") or {}).get("weight")
# A non-dict weight (e.g. a list of multiple specs) means not an OCP
# MXFP4 scheme (e.g. NVFP4 with 2-level scale).
if not isinstance(weight, dict):
return False
return (
weight.get("dtype") == "fp4"
and weight.get("qscheme") == "per_group"
and weight.get("group_size") == 32
)
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
# Quark checkpoints are always handled by QuarkConfig.
if (
isinstance(hf_quant_cfg, dict)
and hf_quant_cfg.get("quant_method") == "quark"
):
return None
if not (
isinstance(hf_quant_cfg, dict)
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
):
return None
model_type = getattr(hf_config, "model_type", None)
if model_type == "deepseek_v4" or user_quant == "deepseek_v4_fp8":
return "deepseek_v4_fp8"
return None
@classmethod
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
# so translate the schema into format Fp8Config.from_config expects.
if config.get("quant_method") == "quark":
quark_exclude = config.get("exclude") or []
config = {
"quant_method": "fp8",
"activation_scheme": "dynamic",
"fmt": "e4m3",
"scale_fmt": "ue8m0",
"weight_block_size": [128, 128],
"ignored_layers": [
name for name in quark_exclude if isinstance(name, str)
],
}
return cast("DeepseekV4FP8Config", super().from_config(config))
def get_quant_method(self, layer, prefix):
if isinstance(layer, RoutedExperts):
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.expert_dtype == "fp4":
if self.moe_quant_algo == "NVFP4":
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4FusedMoE,
)
return ModelOptNvFp4FusedMoE(
quant_config=self._get_nvfp4_config(),
moe_config=layer.moe_config,
)
return Mxfp4MoEMethod(layer.moe_config)
# expert_dtype == "fp8": fall through to Fp8Config which
# returns Fp8MoEMethod with block-wise float32 scales.
return super().get_quant_method(layer, prefix)
-524
View File
@@ -1,524 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 FlashMLA sparse backend, metadata, and metadata builder."""
from dataclasses import dataclass
from typing import Any, ClassVar
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import AttentionSpec
# Pad C128A topk width to this alignment. 128 covers both h_q=64 (B_TOPK=64) and
# h_q=128 (B_TOPK=128). FlashMLA decode asserts extra_topk % B_TOPK == 0;
# unaligned widths (e.g. 17 = ceil(2136/128)) crash the sm100 head64 kernel.
# Padded slots stay -1 and decode_lens caps them via topk_length, so the pad is a
# no-op at kernel level. Mirrors _SPARSE_PREFILL_TOPK_ALIGNMENT in cache_utils.py.
_C128A_TOPK_ALIGNMENT = 128
class DeepseekV4SparseMLABackend(AttentionBackend):
"""DeepSeek-V4 sparse-MLA backend base.
Subclasses ``AttentionBackend`` directly (not the V3.2
``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer
(``DeepseekV4Attention``), so it does not reuse the V3.2 builder or impl, and
only needs to declare its own metadata builder, KV-cache layout, and the
sparse-MLA capability flags.
"""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"fp8_ds_mla",
"fp8", # alias for fp8_ds_mla
]
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [256]
@staticmethod
def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]:
return DeepseekV4SparseMLAMetadataBuilder
@staticmethod
def get_impl_cls() -> type[Any]:
# DeepSeek-V4 runs its attention through ``DeepseekV4Attention.forward``,
# not the generic ``Attention``/``MLAAttention`` layer, so the backend's
# impl class is never instantiated.
raise NotImplementedError(
"DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 "
"attention runs through DeepseekV4Attention."
)
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
# DeepSeek V4 layout: 448 NoPE + 64 RoPE = 512.
return [512]
@classmethod
def is_mla(cls) -> bool:
return True
@classmethod
def is_sparse(cls) -> bool:
return True
@classmethod
def supports_sink(cls) -> bool:
return True
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major in [9, 10]
@dataclass
class DeepseekV4FlashMLAMetadata(AttentionMetadata):
num_reqs: int
max_query_len: int
max_seq_len: int
num_actual_tokens: int # Number of tokens excluding padding.
query_start_loc: torch.Tensor
slot_mapping: torch.Tensor
block_table: torch.Tensor
req_id_per_token: torch.Tensor
block_size: int
topk_tokens: int
# Pre-computed C128A metadata (compress_ratio == 128 only).
# Decode: global slot ids + valid-entry counts (fused from positions).
c128a_global_decode_topk_indices: torch.Tensor | None = None
c128a_decode_topk_lens: torch.Tensor | None = None
# Prefill: local topk indices (used by combine_topk_swa_indices).
c128a_prefill_topk_indices: torch.Tensor | None = None
class DeepseekV4SparseMLAMetadataBuilder(
AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata]
):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.model_config = vllm_config.model_config
# Classify single-token queries (plus num_speculative_tokens via
# supports_spec_as_decode=True) as decodes; longer queries go to prefill.
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
self.topk_tokens = self.model_config.hf_config.index_topk
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.req_id_per_token_buffer = torch.empty(
(max_num_batched_tokens,), dtype=torch.int32, device=device
)
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
self.compress_ratio = self.kv_cache_spec.tokens_per_state
# Pre-allocate compressed slot mapping buffer for CUDA graph address
# stability when compress_ratio > 1.
if self.compress_ratio > 1:
self.compressed_slot_mapping_buffer = torch.empty(
max_num_batched_tokens, dtype=torch.int64, device=device
)
# Pre-allocate C128A topk buffers for CUDA graph address stability.
if self.compress_ratio == 128:
c128a_max_compressed = cdiv(
self.model_config.max_model_len, self.compress_ratio
)
c128a_max_compressed = (
cdiv(c128a_max_compressed, _C128A_TOPK_ALIGNMENT)
* _C128A_TOPK_ALIGNMENT
)
# Stored so _build_c128a_metadata passes it as the kernel's
# max_compressed_tokens, matching the buffer stride. Otherwise the
# kernel's default 8192 iterates past row width and spills writes
# into adjacent rows (present in both decode and prefill branches of
# _build_c128a_topk_metadata_kernel).
self.c128a_max_compressed = c128a_max_compressed
self.c128a_global_decode_buffer = torch.empty(
(max_num_batched_tokens, c128a_max_compressed),
dtype=torch.int32,
device=device,
)
self.c128a_decode_lens_buffer = torch.empty(
max_num_batched_tokens, dtype=torch.int32, device=device
)
self.c128a_prefill_buffer = torch.empty(
(max_num_batched_tokens, c128a_max_compressed),
dtype=torch.int32,
device=device,
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> DeepseekV4FlashMLAMetadata:
cm = common_attn_metadata
req_id_per_token = cm.token_to_req_indices(self.req_id_per_token_buffer)
slot_mapping = cm.slot_mapping
if self.compress_ratio > 1:
slot_mapping = get_compressed_slot_mapping(
cm.num_actual_tokens,
cm.slot_mapping,
cm.query_start_loc,
cm.seq_lens,
cm.block_table_tensor.clamp_(min=0),
int(self.kv_cache_spec.num_states),
self.compress_ratio,
out=self.compressed_slot_mapping_buffer,
)
c128a_fields: dict[str, torch.Tensor | None] = {}
if self.compress_ratio == 128:
c128a_fields = self._build_c128a_metadata(cm, req_id_per_token)
return DeepseekV4FlashMLAMetadata(
num_reqs=cm.num_reqs,
max_query_len=cm.max_query_len,
max_seq_len=cm.max_seq_len,
num_actual_tokens=cm.num_actual_tokens,
query_start_loc=cm.query_start_loc,
slot_mapping=slot_mapping,
block_table=cm.block_table_tensor,
req_id_per_token=req_id_per_token,
block_size=self.kv_cache_spec.block_size,
topk_tokens=self.topk_tokens,
c128a_global_decode_topk_indices=c128a_fields.get(
"c128a_global_decode_topk_indices"
),
c128a_decode_topk_lens=c128a_fields.get("c128a_decode_topk_lens"),
c128a_prefill_topk_indices=c128a_fields.get("c128a_prefill_topk_indices"),
)
def _build_c128a_metadata(
self,
cm: CommonAttentionMetadata,
req_id_per_token: torch.Tensor,
) -> dict[str, torch.Tensor | None]:
"""Pre-compute C128A topk indices for DeepseekV4 (compress_ratio >= 128)."""
# Must match SWA's decode split (no `require_uniform=True`) so
# `c128a_global_decode_topk_indices.shape[0]` lines up with q in
# `_forward_decode`. The per-token C128A kernel handles non-uniform
# query lengths.
(num_decodes, _, num_decode_tokens, num_prefill_tokens) = (
split_decodes_and_prefills(
cm,
decode_threshold=self.reorder_batch_threshold or 1,
)
)
num_total = num_decode_tokens + num_prefill_tokens
if num_total == 0:
return {}
assert cm.positions is not None, (
"positions is required for C128A metadata build"
)
active_topk_width = min(
max(
triton.next_power_of_2(max(cm.max_seq_len // self.compress_ratio, 1)),
_C128A_TOPK_ALIGNMENT,
),
self.c128a_max_compressed,
)
assert active_topk_width >= cm.max_seq_len // self.compress_ratio
assert active_topk_width % _C128A_TOPK_ALIGNMENT == 0
block_size = self.kv_cache_spec.block_size // self.compress_ratio
global_decode, decode_lens, prefill_local = build_c128a_topk_metadata(
cm.positions[:num_total],
self.compress_ratio,
num_decode_tokens,
req_id_per_token,
cm.block_table_tensor[:num_decodes],
block_size,
cm.slot_mapping,
self.c128a_global_decode_buffer,
self.c128a_decode_lens_buffer,
self.c128a_prefill_buffer,
max_compressed_tokens=active_topk_width,
)
result: dict[str, torch.Tensor | None] = {}
if num_decode_tokens > 0:
result["c128a_global_decode_topk_indices"] = global_decode.view(
num_decode_tokens, 1, -1
)
result["c128a_decode_topk_lens"] = decode_lens
if num_prefill_tokens > 0:
result["c128a_prefill_topk_indices"] = prefill_local
return result
class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "FLASHMLA_SPARSE_DSV4"
@staticmethod
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
return DeepseekV4FlashMLAMetadataBuilder
def build_c128a_topk_metadata(
positions: torch.Tensor,
compress_ratio: int,
num_decode_tokens: int,
token_to_req_indices: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
slot_mapping: torch.Tensor,
global_decode_buffer: torch.Tensor,
decode_lens_buffer: torch.Tensor,
prefill_buffer: torch.Tensor,
max_compressed_tokens: int = 8192,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Single kernel for all C128A tokens (decode + prefill).
Decode tokens: position → block_table lookup → global slot ids + topk_lens.
Prefill tokens: position → local indices [0, ..., n-1, -1, ...].
Writes into pre-allocated buffers for CUDA graph address stability.
Returns views of the buffers.
"""
num_tokens = positions.shape[0]
num_prefill_tokens = num_tokens - num_decode_tokens
assert max_compressed_tokens % _C128A_TOPK_ALIGNMENT == 0
assert (
0
< max_compressed_tokens
<= min(global_decode_buffer.shape[1], prefill_buffer.shape[1])
)
assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1
# TODO: support adaptive-width decode on SM120 (needs the FlashInfer
# SM120 kernel to accept a real row stride for eidx).
capability = current_platform.get_device_capability()
if capability is not None and capability.major == 12:
global_decode = global_decode_buffer[:num_decode_tokens]
else:
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
decode_lens = decode_lens_buffer[:num_decode_tokens]
prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens]
assert global_decode.stride(0) == global_decode_buffer.stride(0)
assert prefill_local.stride(0) == prefill_buffer.stride(0)
if num_tokens == 0:
return global_decode, decode_lens, prefill_local
_BUILD_C128A_TOPK_METADATA_KERNEL(
global_decode_buffer,
decode_lens_buffer,
prefill_buffer,
positions,
compress_ratio,
max_compressed_tokens,
num_decode_tokens,
token_to_req_indices,
block_table,
block_size,
slot_mapping,
)
return global_decode, decode_lens, prefill_local
class BuildC128ATopkMetadataKernel(
VllmTritonJitKernel["BuildC128ATopkMetadataKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
compress_ratio: int
max_compressed_tokens: int
block_size: int
triton_block_size: int
@staticmethod
@triton.jit(
do_not_specialize=[
"global_decode_stride",
"prefill_local_stride",
"num_decode_tokens",
"block_table_stride",
]
)
def kernel(
# Decode outputs
global_decode_ptr,
global_decode_stride,
decode_lens_ptr,
# Prefill output
prefill_local_ptr,
prefill_local_stride,
# Inputs
positions_ptr,
compress_ratio,
max_compressed_tokens,
num_decode_tokens,
token_to_req_indices_ptr,
block_table_ptr,
block_table_stride,
block_size,
slot_mapping_ptr,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
position = tl.load(positions_ptr + token_idx)
num_compressed = (position + 1) // compress_ratio
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
is_decode = token_idx < num_decode_tokens
if is_decode:
# --- Decode: block-table lookup → global slot ids + count ---
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
count = tl.zeros((), dtype=tl.int32)
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
is_valid = offset < num_compressed
block_indices = offset // block_size
block_numbers = tl.load(
block_table_ptr + req_idx * block_table_stride + block_indices,
mask=mask & is_valid,
)
block_offsets = offset % block_size
slot_ids = block_numbers * block_size + block_offsets
slot_ids = tl.where(is_valid, slot_ids, -1)
tl.store(
global_decode_ptr + token_idx * global_decode_stride + offset,
slot_ids,
mask=mask,
)
count += tl.sum(is_valid.to(tl.int32), axis=0)
tl.store(
decode_lens_ptr + token_idx,
tl.where(is_valid_token, count, 0),
)
else:
# --- Prefill: write local indices ---
pfx_idx = token_idx - num_decode_tokens
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
tl.store(
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
tl.where(offset < num_compressed, offset, -1),
mask=mask,
)
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if getattr(hf_config, "model_type", None) != "deepseek_v4":
return []
compress_ratio = 128
max_model_len = vllm_config.model_config.max_model_len
if max_model_len <= 0:
return []
max_compressed_tokens = cdiv(max_model_len, compress_ratio)
max_compressed_tokens = (
cdiv(max_compressed_tokens, _C128A_TOPK_ALIGNMENT) * _C128A_TOPK_ALIGNMENT
)
return self._trace_dispatch(self.dispatch)(
compress_ratio=compress_ratio,
max_compressed_tokens=max_compressed_tokens,
# DeepSeek V4 sparse MLA uses 256-token KV pages; C128A metadata
# works in compressed-token units.
block_size=256 // compress_ratio,
triton_block_size=1024,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
global_decode_buffer=TritonWarmupTensor(
torch.int32, shape=(1, 1), strides=(1, 1)
),
decode_lens_buffer=int32_ptr,
prefill_buffer=TritonWarmupTensor(
torch.int32, shape=(1, 1), strides=(1, 1)
),
positions=TritonWarmupTensor(torch.int64),
compress_ratio=compile_key.compress_ratio,
max_compressed_tokens=compile_key.max_compressed_tokens,
num_decode_tokens=0,
token_to_req_indices=int32_ptr,
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
block_size=compile_key.block_size,
slot_mapping=TritonWarmupTensor(torch.int64),
)
@kernel_launcher
def __call__(
self,
global_decode_buffer: torch.Tensor,
decode_lens_buffer: torch.Tensor,
prefill_buffer: torch.Tensor,
positions: torch.Tensor,
compress_ratio: int,
max_compressed_tokens: int,
num_decode_tokens: int,
token_to_req_indices: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
slot_mapping: torch.Tensor,
) -> LaunchSpec:
return (positions.shape[0],), dict(
global_decode_ptr=global_decode_buffer,
global_decode_stride=global_decode_buffer.stride(0),
decode_lens_ptr=decode_lens_buffer,
prefill_local_ptr=prefill_buffer,
prefill_local_stride=prefill_buffer.stride(0),
block_table_stride=block_table.stride(0),
BLOCK_SIZE=1024,
)
_BUILD_C128A_TOPK_METADATA_KERNEL = BuildC128ATopkMetadataKernel()
-14
View File
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stub for platforms where the DeepSeek-V4 vision variant is unsupported."""
from torch import nn
class DeepseekV4ForConditionalGeneration(nn.Module):
def __init__(self, *, vllm_config, prefix: str = ""):
super().__init__()
raise NotImplementedError(
"DeepSeek-V4 vision (DeepseekV4ForConditionalGeneration) is not "
"supported on the current platform; use CUDA or ROCm."
)
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-419
View File
@@ -1,419 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark draft model for DeepSeek-V4 on Intel XPU.
Minimal XPU port of nvidia/dspark.py. Replaces tilelang MHC kernels with
the platform-agnostic custom ops (HCHeadOp, MHCPostOp) already used by the
XPU MTP path, and uses the XPU Triton-based qnorm_rope_kv_fp8_insert for
context KV precomputation.
"""
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp, MHCPostOp
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.utils import maybe_prefix
from .model import (
DeepseekV4DecoderLayer,
make_deepseek_v4_expert_params_mapping,
)
logger = init_logger(__name__)
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DSparkDeepseekV4Model(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
self.hidden_size = config.hidden_size
self.hc_mult = config.hc_mult
self.hc_eps = config.hc_eps
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
# Shared with target (aliased by speculator loading utility).
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.main_proj = ReplicatedLinear(
config.hidden_size * len(self.target_layer_ids),
config.hidden_size,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "main_proj"),
)
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
current_vllm_config = get_current_vllm_config()
self.layers = nn.ModuleList(
[
DeepseekV4DecoderLayer(
current_vllm_config,
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
)
for i in range(self.num_dspark_layers)
]
)
# Heads: final norm + hc_head, and the Markov head
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32), requires_grad=False
)
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
# XPU MHC ops (replaces tilelang)
self.mhc_post_op = MHCPostOp()
self.hc_head_op = HCHeadOp()
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
"""main_x = main_norm(main_proj(concat of target aux hidden states))."""
return self.main_norm(self.main_proj(aux_hidden_states))
@torch.inference_mode()
def precompute_and_store_context_kv(
self,
main_x: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
"""Insert the sliding-window context KV for every draft layer.
Each layer derives its context KV from the SAME projected target hidden
``main_x``, via that layer's own wkv + kv_norm + RoPE + quant, then
writes it at the layer's context slots.
"""
for i, layer in enumerate(self.layers):
slot_mapping = (
None if context_slot_mappings is None else context_slot_mappings[i]
)
attn = layer.attn
# wkv part of the fused wq_a|wkv projection (q_lora part discarded)
qr_kv, _ = attn.fused_wqa_wkv(main_x)
kv = qr_kv[..., attn.q_lora_rank :]
kv = attn.kv_norm(kv)
if slot_mapping is None:
continue
_insert_context_kv(attn, kv, context_positions, slot_mapping)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
residual = post_mix = res_mix = None
for layer in self.layers:
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
)
# mhc_post: merge hyper-connection copies
hidden_states = self.mhc_post_op(hidden_states, residual, post_mix, res_mix)
# hc_head: reduces hc copies; return pre-norm head hidden
hidden_states = self.hc_head_op(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
self.hc_head_base,
self.rms_norm_eps,
self.hc_eps,
)
return hidden_states
def _insert_context_kv(
attn: nn.Module,
kv: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
On XPU, we reuse the same xpu_qnorm_rope_kv_fp8_insert kernel used in
the forward path, passing a dummy q (result discarded).
"""
from vllm.models.deepseek_v4.xpu.xpu_qnorm_rope_kv_fp8_insert import (
xpu_qnorm_rope_kv_fp8_insert,
)
swa_cache = attn.swa_cache_layer.kv_cache
block_size = attn.swa_cache_layer.block_size
cos_sin_cache = attn.rotary_emb.cos_sin_cache
n_ctx = kv.shape[0]
# Dummy q — we only care about the KV insert side effect.
dummy_q = torch.empty(
(n_ctx, attn.n_local_heads, attn.head_dim),
dtype=kv.dtype,
device=kv.device,
)
xpu_qnorm_rope_kv_fp8_insert(
dummy_q,
kv,
swa_cache,
slot_mapping,
positions,
cos_sin_cache,
attn.eps,
block_size,
)
class DSparkDeepseekV4ForCausalLM(nn.Module):
"""XPU DSpark draft model entry point for DeepSeek-V4."""
has_own_embed_tokens = False
has_own_lm_head = False
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
self.model = DSparkDeepseekV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# Shared with the target (aliased by the speculator's load utility).
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
# --- Hooks used by the speculator ---
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
return self.model.combine_hidden_states(aux_hidden_states)
def get_draft_kv_cache_layer_names(self) -> list[str]:
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
def precompute_and_store_context_kv(
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
self.model.precompute_and_store_context_kv(
context_states, context_positions, context_slot_mappings
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model(input_ids, positions, inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Base logits U_k = lm_head(norm(head_hidden))."""
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.compute_logits(hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
return draft_ids # full-vocab: draft ids are target ids
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
# --- Weight loading ---
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load ``mtp.{0,1,2}.*`` draft weights from the target checkpoint."""
first_layer = self.model.layers[0]
use_mega_moe = first_layer.ffn.use_mega_moe
if use_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
else:
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
)
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
stacked_params_mapping = [
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
continue
name = mapped
# .scale -> per-method scale suffix
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
# Expert weights
if ".experts." in name:
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
if name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
success = param.weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(name_mapped)
break
continue
# Stacked params (decoder-layer only)
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
break
param = params_dict[name]
param.weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
break
else:
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
)
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
self._finalize_moe()
logger.info_once("DSpark XPU draft model loaded: %d params", len(loaded_params))
return loaded_params
def _finalize_moe(self) -> None:
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map checkpoint ``mtp.{i}.*`` name to this model's parameter path."""
m = re.match(r"mtp\.(\d+)\.(.*)", name)
if m is None:
return None
stage = int(m.group(1))
rest = m.group(2)
if rest.startswith("confidence_head."):
return None
head_prefixes = (
"norm.",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
"markov_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
):
return f"model.{rest}"
return f"model.layers.{stage}.{rest}"
File diff suppressed because it is too large Load Diff
-510
View File
@@ -1,510 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
pieces that have no analogue in V3/V32:
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
the fused ``eh_proj``);
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
* V4-specific checkpoint weight-name remapping in ``load_weights``.
"""
import typing
from collections.abc import Callable, Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .model import (
DeepseekV4DecoderLayer,
make_deepseek_v4_expert_params_mapping,
)
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which FusedMoEFactory method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
# Other FP8 linear scales (including shared experts) always use
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
topk_indices_buffer: torch.Tensor,
prefix: str,
aux_stream_list: list | None = None,
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
self.rms_norm_eps = config.rms_norm_eps
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
# fusing them the way V3 does with eh_proj.
self.e_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.e_proj",
)
self.h_proj = ReplicatedLinear(
config.hidden_size,
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.h_proj",
)
self.hc_eps = config.hc_eps
self.hc_mult = config.hc_mult
self.hc_dim = self.hc_mult * config.hidden_size
self.hc_head_fn = nn.Parameter(
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(self.hc_mult, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
self.mtp_block = DeepseekV4DecoderLayer(
vllm_config,
prefix,
topk_indices_buffer=topk_indices_buffer,
aux_stream_list=aux_stream_list,
)
self.hc_head_op = HCHeadOp()
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
# reshape to (T, hc_mult, D) — the training-time layout — before
# the fused norm pass so both inputs are 3D-friendly.
previous_hidden_states = previous_hidden_states.view(
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
self.enorm.weight.data,
self.hnorm.weight.data,
self.enorm.variance_epsilon,
self.hc_mult,
)
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
inputs_embeds
).unsqueeze(-2)
hidden_states, residual, post_mix, res_mix = self.mtp_block(
positions=positions, x=hidden_states, input_ids=None
)
hidden_states = self.mtp_block.hc_post(
hidden_states, residual, post_mix, res_mix
)
# Return the flat pre-hc_head residual so it can be re-fed as the
# next spec step's `previous_hidden_states` when
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
return hidden_states.flatten(1)
class DeepSeekV4MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
self.device = current_platform.device_type
topk_tokens = config.index_topk
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
topk_tokens,
dtype=torch.int32,
device=self.device,
)
# Disable aux streams on XPU — no multi-stream overlap support.
aux_stream_list = None
# to map the exact layer index from weights
self.layers = torch.nn.ModuleDict(
{
str(idx): DeepSeekV4MultiTokenPredictorLayer(
vllm_config,
self.topk_indices_buffer,
f"{prefix}.layers.{idx}",
aux_stream_list=aux_stream_list,
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
# hc_head here so logits are computed from the dense hidden state.
hidden_states = hidden_states.view(
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
)
hidden_states = mtp_layer.hc_head_op(
hidden_states,
mtp_layer.hc_head_fn,
mtp_layer.hc_head_scale,
mtp_layer.hc_head_base,
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
)
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
return logits
class DeepSeekV4MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
self.model = DeepSeekV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Weight name remapping for checkpoint compatibility.
# Maps checkpoint weight paths to model parameter paths.
WEIGHT_NAME_REMAPPING: dict[str, str] = {
".emb.tok_emb.weight": ".embed_tokens.weight",
".head.weight": ".shared_head.head.weight",
".norm.weight": ".shared_head.norm.weight",
}
def _remap_weight_name(name: str) -> str:
"""Remap checkpoint weight names to model parameter names."""
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
if old_pattern in name:
name = name.replace(old_pattern, new_pattern)
return name
def _find_mtp_layer_idx(name: str) -> int:
subnames = name.split(".")
for subname in subnames:
try:
# we return the first encountered integer
return int(subname)
except ValueError:
continue
return 0
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
else:
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
)
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
# for the rename below based on the model's expert dtype.
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
for name, loaded_weight in weights:
mtp_layer_idx = _find_mtp_layer_idx(name)
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
# `model.layers.{num_hidden_layers + i}.*` so that
# get_spec_layer_idx_from_weight_name can identify them.
name = name.replace(
f"mtp.{mtp_layer_idx}.",
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
)
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
name = _remap_weight_name(name)
name = self._rewrite_spec_layer_name(spec_layer, name)
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
continue
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
continue
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
break
else:
if ".experts." in name:
# Reinterpret E8M0 scales as uint8 to preserve raw
# exponent bytes; numeric copy_() would zero them.
# Mirrors the main DeepseekV4 loader.
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for mapping in expert_mapping:
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
loaded_params.add(name_mapped)
break
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
)
if name.endswith(".ffn.gate.bias"):
# ``e_score_correction_bias`` lives on the gate
# under a different attribute name.
name = name.replace(
".ffn.gate.bias",
".ffn.gate.e_score_correction_bias",
)
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
loaded_layers: set[int] = set()
for param_name in loaded_params:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
if spec_layer is not None:
loaded_layers.add(spec_layer)
for layer_idx in range(
self.model.mtp_start_layer_idx,
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
):
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
raise ValueError(
f"MTP speculative decoding layer {layer_idx} weights "
f"missing from checkpoint. The checkpoint may have "
f"been quantized without including the MTP layers. "
f"Use a checkpoint that includes MTP layer weights, "
f"or disable speculative decoding."
)
self.finalize_mega_moe_weights()
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def finalize_mega_moe_weights(self) -> None:
for layer in self.model.layers.values():
layer.mtp_block.ffn.finalize_mega_moe_weights()
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
and rename shared layer weights to be top level.
"""
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"h_proj",
"e_proj",
"shared_head",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
]
shared_weight_names = ["embed_tokens"]
spec_layer_weight = False
shared_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
if weight_name in shared_weight_names:
shared_weight = True
break
if not spec_layer_weight:
# treat rest weights as weights for transformer layer block
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
elif shared_weight:
# treat shared weights as top level weights
name = name.replace(f"model.layers.{spec_layer}.", "model.")
return name
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""XPU Triton replacement for fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert.
Does: Q per-head RMSNorm + GPT-J RoPE, KV GPT-J RoPE + UE8M0 FP8 quant + insert.
Uses the existing quantize_and_insert_k_cache for the FP8 portion.
"""
import torch
from vllm.triton_utils import tl, triton
HEAD_DIM = 512
ROPE_DIM = 64
NOPE_DIM = HEAD_DIM - ROPE_DIM
HALF_ROPE = ROPE_DIM // 2
@triton.jit
def _xpu_qnorm_rope_kernel(
q_ptr, # [num_tokens, num_heads, HEAD_DIM]
kv_ptr, # [num_tokens, HEAD_DIM]
kv_out_ptr, # [num_tokens, HEAD_DIM] bf16 (RoPE-applied kv for cache insert)
position_ids_ptr,
cos_sin_cache_ptr,
eps: tl.constexpr,
num_tokens,
num_heads: tl.constexpr,
HEAD_DIM: tl.constexpr,
ROPE_DIM: tl.constexpr,
NOPE_DIM: tl.constexpr,
HALF_ROPE: tl.constexpr,
):
"""Apply per-head RMSNorm + GPT-J RoPE on Q, GPT-J RoPE on KV.
GPT-J interleaved format: pairs are (data[2i], data[2i+1]).
cos_sin_cache layout: [max_pos, ROPE_DIM] with first HALF_ROPE=cos,
second HALF_ROPE=sin.
"""
token_idx = tl.program_id(0)
head_idx = tl.program_id(1)
if token_idx >= num_tokens:
return
pos = tl.load(position_ids_ptr + token_idx).to(tl.int64)
# Load cos/sin for this position
rope_pair_idx = tl.arange(0, HALF_ROPE)
cos_val = tl.load(cos_sin_cache_ptr + pos * ROPE_DIM + rope_pair_idx).to(tl.float32)
sin_val = tl.load(
cos_sin_cache_ptr + pos * ROPE_DIM + HALF_ROPE + rope_pair_idx
).to(tl.float32)
if head_idx < num_heads:
# ========== Q: per-head RMSNorm + GPT-J RoPE ==========
q_base = q_ptr + token_idx * num_heads * HEAD_DIM + head_idx * HEAD_DIM
# Load full head
offs = tl.arange(0, HEAD_DIM)
q_vals = tl.load(q_base + offs).to(tl.float32)
# RMSNorm (no weight)
sq_sum = tl.sum(q_vals * q_vals, axis=0)
rms = tl.rsqrt(sq_sum / HEAD_DIM + eps)
q_vals = q_vals * rms
# Store ONLY the NoPE portion (positions 0..NOPE_DIM-1)
nope_mask = offs < NOPE_DIM
tl.store(q_base + offs, q_vals.to(q_ptr.type.element_ty), mask=nope_mask)
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions:
even_offs = NOPE_DIM + rope_pair_idx * 2
odd_offs = NOPE_DIM + rope_pair_idx * 2 + 1
# Re-load original values at rope positions and normalize
q_even = tl.load(q_base + even_offs).to(tl.float32) * rms
q_odd = tl.load(q_base + odd_offs).to(tl.float32) * rms
new_even = q_even * cos_val - q_odd * sin_val
new_odd = q_even * sin_val + q_odd * cos_val
# Store rotated RoPE values
tl.store(q_base + even_offs, new_even.to(q_ptr.type.element_ty))
tl.store(q_base + odd_offs, new_odd.to(q_ptr.type.element_ty))
else:
# ========== KV: GPT-J RoPE only ==========
kv_base = kv_ptr + token_idx * HEAD_DIM
kv_out_base = kv_out_ptr + token_idx * HEAD_DIM
# Copy only the NoPE portion; the RoPE region is written exclusively
# by the rotated stores below. Masking avoids a write-after-write
# hazard on the overlapping RoPE addresses (matches the Q branch).
offs = tl.arange(0, HEAD_DIM)
kv_full = tl.load(kv_base + offs)
nope_mask = offs < NOPE_DIM
tl.store(kv_out_base + offs, kv_full, mask=nope_mask)
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions
even_offs = NOPE_DIM + rope_pair_idx * 2
odd_offs = NOPE_DIM + rope_pair_idx * 2 + 1
kv_even = tl.load(kv_base + even_offs).to(tl.float32)
kv_odd = tl.load(kv_base + odd_offs).to(tl.float32)
new_even = kv_even * cos_val - kv_odd * sin_val
new_odd = kv_even * sin_val + kv_odd * cos_val
tl.store(kv_out_base + even_offs, new_even.to(kv_out_ptr.type.element_ty))
tl.store(kv_out_base + odd_offs, new_odd.to(kv_out_ptr.type.element_ty))
def xpu_qnorm_rope_kv_fp8_insert(
q: torch.Tensor, # [num_tokens, num_heads, HEAD_DIM] bf16, in-place
kv: torch.Tensor, # [num_tokens, HEAD_DIM] bf16
swa_kv_cache: torch.Tensor, # [num_blocks, block_size, 584] or flat uint8
slot_mapping: torch.Tensor, # [num_tokens] int64
positions: torch.Tensor, # [num_tokens] int64
cos_sin_cache: torch.Tensor, # [max_pos, ROPE_DIM]
eps: float,
block_size: int,
):
"""XPU Triton: qnorm+rope on Q, rope on KV, then FP8 UE8M0 quant+insert."""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
quantize_and_insert_k_cache,
)
num_tokens = q.shape[0]
num_heads = q.shape[1]
# Allocate temp buffer for RoPE-applied KV
kv_roped = torch.empty_like(kv)
# Grid: one program per (token, head_or_kv)
# head_idx < num_heads: process Q head
# head_idx == num_heads: process KV
grid = (num_tokens, num_heads + 1)
_xpu_qnorm_rope_kernel[grid](
q,
kv,
kv_roped,
positions,
cos_sin_cache,
eps,
num_tokens,
num_heads=num_heads,
HEAD_DIM=HEAD_DIM,
ROPE_DIM=ROPE_DIM,
NOPE_DIM=NOPE_DIM,
HALF_ROPE=HALF_ROPE,
)
# FP8 UE8M0 quant + paged insert (reuse existing Triton kernel)
# swa_kv_cache may be [num_blocks, block_size, 584] or [num_blocks, flat]
# quantize_and_insert_k_cache expects [num_blocks, block_bytes] uint8
cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
quantize_and_insert_k_cache(
kv_roped,
cache_2d,
slot_mapping,
block_size=block_size,
)
-368
View File
@@ -1,368 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""XPU DeepSeek-V4 attention subclass.
Subclasses the shared ``DeepseekV4Attention`` ABC and provides XPU-native
Triton kernels for decode (FP8 dequant + BF16 attention) and prefill
(BF16 gathered KV + sparse attention).
"""
from typing import TYPE_CHECKING, cast
import torch
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
)
from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import (
xpu_sparse_decode_fp8,
)
from vllm.v1.attention.ops.xpu_mla_sparse import triton_bf16_mla_sparse_interface
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "XPU_V4_MLA_SPARSE"
class DeepseekV4XPUAttention(DeepseekV4Attention):
"""XPU sparse MLA attention layer for DeepSeek V4."""
backend_cls = DeepseekV4XPUSparseBackend
use_flashmla_fp8_layout = True
def __init__(self, *args, **kwargs) -> None:
# torch.cuda.Event() raises RuntimeError on XPU ("dummy base class").
# The Base and DeepseekV4Indexer both create cuda Events in __init__, so
# we temporarily redirect torch.cuda.Event → torch.xpu.Event.
_orig_event = torch.cuda.Event
torch.cuda.Event = torch.xpu.Event # type: ignore[misc]
try:
super().__init__(*args, **kwargs)
finally:
torch.cuda.Event = _orig_event # type: ignore[misc]
def _fused_qnorm_rope_kv_insert(self, q, kv, positions, attn_metadata):
from typing import cast
if not isinstance(attn_metadata, dict):
# Profile run: no-op, just return q (no padding needed on XPU).
return q
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
from vllm.models.deepseek_v4.xpu.xpu_qnorm_rope_kv_fp8_insert import (
xpu_qnorm_rope_kv_fp8_insert,
)
xpu_qnorm_rope_kv_fp8_insert(
q,
kv,
self.swa_cache_layer.kv_cache,
swa_metadata.slot_mapping,
positions,
self.rotary_emb.cos_sin_cache,
self.eps,
swa_metadata.block_size,
)
return q
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return num_heads
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
o_fp8, o_scale = fused_inv_rope_fp8_quant(
o,
positions,
self.rotary_emb.cos_sin_cache,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
tma_aligned_scales=False,
)
# Precomputed contiguous [G, K, N] weight and [G, K/bs, N/bs] scale.
wo_a_weight = self.wo_a.bmm_weight
wo_a_scale = self.wo_a.bmm_scale
# TODO: optimize fused_inv_rope_fp8_quant for xpu bmm to
# eliminate o_scale transpose + contiguous
z = torch.ops.vllm.xpu_fp8_bmm(
o_fp8.transpose(0, 1),
wo_a_weight,
torch.bfloat16,
o_scale.transpose(0, 1).contiguous(),
wo_a_scale,
None,
)
return self.wo_b(z.transpose(0, 1).flatten(1))
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
assert output.shape == q.shape, (
f"output buffer shape {output.shape} must match q shape {q.shape}"
)
assert output.dtype == q.dtype, (
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
)
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# Warmup dummy run: reserve workspace, skip actual kernels.
swa_only = self.compress_ratio <= 1
N = (
0
if swa_only
else (self.max_model_len + self.compress_ratio - 1)
// self.compress_ratio
)
M = N + self.window_size + self.max_num_batched_tokens
current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
)
output.zero_()
return
assert isinstance(attn_metadata, dict)
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_only = self.compress_ratio <= 1
self_kv_cache = self.kv_cache if not swa_only else None
swa_kv_cache = self.swa_cache_layer.kv_cache
# Split prefill and decode
num_decodes = swa_metadata.num_decodes
num_prefills = swa_metadata.num_prefills
num_decode_tokens = swa_metadata.num_decode_tokens
if num_prefills > 0:
self._forward_prefill(
q=q[num_decode_tokens:],
positions=positions[num_decode_tokens:],
compressed_k_cache=self_kv_cache,
swa_k_cache=swa_kv_cache,
output=output[num_decode_tokens:],
attn_metadata=flashmla_metadata,
swa_metadata=swa_metadata,
)
if num_decodes > 0:
self._forward_decode(
q=q[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=flashmla_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
)
def _forward_decode(
self,
q: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_metadata: "DeepseekSparseSWAMetadata",
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
output: torch.Tensor,
) -> None:
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
topk_indices = None
topk_lens = None
if not swa_only:
assert attn_metadata is not None
assert swa_metadata.is_valid_token is not None
block_size = attn_metadata.block_size // self.compress_ratio
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
if self.compress_ratio == 4:
# C4A: local indices differ per layer (filled by Indexer).
assert self.topk_indices_buffer is not None
global_indices, topk_lens = compute_global_topk_indices_and_lens(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
block_size,
is_valid,
)
topk_indices = global_indices.view(num_decode_tokens, 1, -1)
else:
# C128A: pre-computed during metadata build.
topk_indices = attn_metadata.c128a_global_decode_topk_indices
topk_lens = attn_metadata.c128a_decode_topk_lens
swa_indices = swa_metadata.decode_swa_indices
swa_lens = swa_metadata.decode_swa_lens
assert swa_indices is not None and swa_lens is not None
xpu_sparse_decode_fp8(
q=q,
kv_cache=kv_cache,
swa_kv_cache=self.swa_cache_layer.kv_cache,
swa_only=swa_only,
topk_indices=topk_indices,
topk_lens=topk_lens,
swa_indices=swa_indices,
swa_lens=swa_lens,
attn_sink=self.attn_sink,
softmax_scale=self.scale,
head_dim=self.head_dim,
nope_head_dim=self.nope_head_dim,
rope_head_dim=self.rope_head_dim,
out=output,
)
def _forward_prefill(
self,
q: torch.Tensor,
positions: torch.Tensor,
compressed_k_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
output: torch.Tensor,
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
) -> None:
swa_only = attn_metadata is None
num_prefills = swa_metadata.num_prefills
num_prefill_tokens = swa_metadata.num_prefill_tokens
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
# Use pre-computed prefill metadata.
seq_lens = swa_metadata.prefill_seq_lens
gather_lens = swa_metadata.prefill_gather_lens
assert seq_lens is not None
assert gather_lens is not None
# Derive prefill-local token offsets from the full query_start_loc_cpu.
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
assert query_start_loc_cpu is not None
assert query_start_loc is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
if not swa_only:
if self.compress_ratio == 4:
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
topk_indices = topk_indices[:num_prefill_tokens]
else:
# C128A: pre-computed during metadata build.
assert attn_metadata is not None
topk_indices = attn_metadata.c128a_prefill_topk_indices
top_k = topk_indices.shape[-1]
N = (self.max_model_len + self.compress_ratio - 1) // self.compress_ratio
else:
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
top_k = 0
N = 0
M = N + self.window_size + self.max_num_batched_tokens
chunk_size_const = self.PREFILL_CHUNK_SIZE
num_chunks = (num_prefills + chunk_size_const - 1) // chunk_size_const
workspace_manager = current_workspace_manager()
kv = workspace_manager.get_simultaneous(
((chunk_size_const, M, q.shape[-1]), torch.bfloat16),
)[0]
for chunk_idx in range(num_chunks):
chunk_start = chunk_idx * chunk_size_const
chunk_end = min(chunk_start + chunk_size_const, num_prefills)
chunk_size = chunk_end - chunk_start
if not swa_only:
# Gather compressed KV
assert attn_metadata is not None
block_table = attn_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
compressed_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
gather_lens=None,
block_table=block_table[chunk_start:chunk_end],
block_size=attn_metadata.block_size // self.compress_ratio,
offset=0,
)
# Gather SWA KV
swa_block_table = swa_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
swa_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end],
gather_lens=gather_lens[chunk_start:chunk_end],
block_table=swa_block_table[chunk_start:chunk_end],
block_size=swa_metadata.block_size,
offset=N,
)
# Combine the topk indices and SWA indices for gathered KV cache
query_start = (
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
)
query_end = (
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
)
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices[query_start:query_end],
query_start_loc[
num_decodes + chunk_start : num_decodes + chunk_end + 1
],
seq_lens[chunk_start:chunk_end],
gather_lens[chunk_start:chunk_end],
self.window_size,
self.compress_ratio,
top_k,
M,
N,
)
kv_ws = kv[:chunk_size].reshape(-1, 1, q.shape[-1])
out, _, _ = triton_bf16_mla_sparse_interface(
q=q[query_start:query_end],
kv=kv_ws,
indices=combined_indices.unsqueeze(1),
sm_scale=self.scale,
d_v=q.shape[-1],
block_dpe=0,
)
output[query_start:query_end] = out
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""XPU sparse decode for DeepSeek V4 with FP8 KV cache.
Strategy: dequantize FP8 UE8M0 KV cache pages to BF16 on the fly,
then reuse the BF16 sparse MLA attention kernel (xpu_sparse_mla_bf16).
This keeps the external KV cache layout identical to CUDA/ROCm.
"""
import torch
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.xpu_mla_sparse import (
triton_bf16_mla_sparse_interface,
)
# FP8 DS MLA cache layout constants
TOKEN_FP8_DIM = 448 # NoPE portion in FP8
TOKEN_BF16_DIM = 64 # RoPE portion in BF16
TOKEN_SCALE_DIM = 8 # UE8M0 scales per token
QUANT_BLOCK_SIZE = 64 # Elements per quant block
OUTPUT_DIM = 512 # = TOKEN_FP8_DIM + TOKEN_BF16_DIM after dequant
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2 # 576 bytes per token
@triton.jit
def _dequant_gather_slots_kernel(
# Output workspace: [total_slots, OUTPUT_DIM] bf16
out_ptr,
# FP8 paged cache base pointer (uint8 flat)
cache_ptr,
# Global slot indices: [total_slots] int32
indices_ptr,
# Cache geometry
cache_block_size: tl.constexpr,
token_data_size: tl.constexpr, # 576
block_stride: tl.int64, # k_cache.stride(0) — total uint8 per block
fp8_dim: tl.constexpr, # 448
bf16_dim: tl.constexpr, # 64
scale_dim: tl.constexpr, # 8
quant_block: tl.constexpr, # 64
output_dim: tl.constexpr, # 512
n_quant_blocks: tl.constexpr, # 7
):
"""Dequantize scattered FP8 slots into a flat BF16 workspace.
Grid: [total_slots] — one program per slot to gather.
Cache block layout (block_size tokens):
[0, block_size*576): Token data, each token 448 FP8 + 128 BF16
[block_size*576, block_size*576 + block_size*8): Scales
"""
pid = tl.program_id(0)
# Load global slot index
slot_idx = tl.load(indices_ptr + pid).to(tl.int64)
# Output pointer for this slot
out_row_ptr = out_ptr + pid * output_dim
# Handle invalid slots (index < 0): write zeros
if slot_idx < 0:
zero = tl.zeros([quant_block], dtype=tl.bfloat16)
for i in tl.static_range(0, 512, 64):
offsets = i + tl.arange(0, quant_block)
mask = offsets < output_dim
tl.store(out_row_ptr + offsets, zero, mask=mask)
return
# Compute block and position within block
block_idx = slot_idx // cache_block_size
pos_in_block = slot_idx % cache_block_size
# Block base pointer
block_base = cache_ptr + block_idx * block_stride
# Token data: at offset pos_in_block * token_data_size within block
token_data_ptr = block_base + pos_in_block * token_data_size
# Scale: after all token data, at offset
# cache_block_size * token_data_size + pos_in_block * scale_dim
scale_region_offset = tl.cast(cache_block_size, tl.int64) * token_data_size
token_scale_ptr = block_base + scale_region_offset + pos_in_block * scale_dim
# ========== Dequantize FP8 portion (448 elements) ==========
for qblock_idx in tl.static_range(n_quant_blocks):
qblock_start = qblock_idx * quant_block
offsets = qblock_start + tl.arange(0, quant_block)
mask = offsets < fp8_dim
# Load FP8 as uint8 and bitcast
x_uint8 = tl.load(token_data_ptr + offsets, mask=mask, other=0)
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
x_float = x_fp8.to(tl.float32)
# Load UE8M0 scale: scale = 2^(stored_value - 127)
encoded_scale = tl.load(token_scale_ptr + qblock_idx)
exponent = encoded_scale.to(tl.float32) - 127.0
scale = tl.exp2(exponent)
# Dequantize and store as bf16
x_dequant = x_float * scale
tl.store(out_row_ptr + offsets, x_dequant.to(tl.bfloat16), mask=mask)
# ========== Copy BF16 portion (64 elements) directly ==========
bf16_src_ptr = (token_data_ptr + fp8_dim).to(tl.pointer_type(tl.bfloat16))
bf16_out_ptr = (out_row_ptr + fp8_dim).to(tl.pointer_type(tl.bfloat16))
for j in tl.static_range(bf16_dim // 16):
chunk_offsets = j * 16 + tl.arange(0, 16)
bf16_vals = tl.load(bf16_src_ptr + chunk_offsets)
tl.store(bf16_out_ptr + chunk_offsets, bf16_vals)
def dequant_gather_slots(
out: torch.Tensor, # [total_slots, 512] bf16, pre-allocated
cache: torch.Tensor, # [num_blocks, block_size, head_bytes] uint8
indices: torch.Tensor, # [total_slots] int32, global slot IDs
cache_block_size: int, # block_size for this cache
) -> None:
"""Dequantize FP8 UE8M0 pages at scattered slot indices into bf16."""
total_slots = indices.shape[0]
if total_slots == 0:
return
block_stride = cache.stride(0)
_dequant_gather_slots_kernel[(total_slots,)](
out,
cache,
indices,
cache_block_size=cache_block_size,
token_data_size=TOKEN_DATA_SIZE,
block_stride=block_stride,
fp8_dim=TOKEN_FP8_DIM,
bf16_dim=TOKEN_BF16_DIM,
scale_dim=TOKEN_SCALE_DIM,
quant_block=QUANT_BLOCK_SIZE,
output_dim=OUTPUT_DIM,
n_quant_blocks=7,
)
def xpu_sparse_decode_fp8(
q: torch.Tensor, # [num_tokens, num_heads, head_dim]
kv_cache: torch.Tensor | None, # [num_blocks, block_size, head_bytes] uint8
swa_kv_cache: torch.Tensor, # [num_blocks, swa_block_size, head_bytes] uint8
swa_only: bool,
topk_indices: torch.Tensor | None, # [num_tokens, 1, topk] global slot IDs
topk_lens: torch.Tensor | None,
swa_indices: torch.Tensor, # [num_tokens, 1, swa_k] global slot IDs
swa_lens: torch.Tensor,
attn_sink: torch.Tensor,
softmax_scale: float,
head_dim: int,
nope_head_dim: int,
rope_head_dim: int,
out: torch.Tensor, # [num_tokens, num_heads, head_dim]
) -> None:
"""XPU decode: dequant FP8 pages to BF16, then BF16 sparse MLA attention.
Keeps external FP8 KV cache layout identical to CUDA/ROCm.
Performance is slower due to on-the-fly dequant, but correctness is
guaranteed by reusing the validated BF16 attention kernel.
"""
num_tokens = q.shape[0]
device = q.device
if not swa_only and topk_indices is not None and kv_cache is not None:
topk_idx_2d = (
topk_indices.squeeze(1) if topk_indices.dim() == 3 else topk_indices
)
max_topk = topk_idx_2d.shape[1]
else:
topk_idx_2d = None
max_topk = 0
use_topk = topk_idx_2d is not None
swa_idx_2d = swa_indices.squeeze(1) if swa_indices.dim() == 3 else swa_indices
max_swa = swa_idx_2d.shape[1]
K_total = max_topk + max_swa
# Topk rows precede SWA rows in the flat workspace.
n_topk_rows = num_tokens * max_topk
workspace = torch.empty(
(num_tokens * K_total, OUTPUT_DIM), dtype=torch.bfloat16, device=device
)
if topk_idx_2d is not None and kv_cache is not None and n_topk_rows > 0:
dequant_gather_slots(
workspace[:n_topk_rows],
kv_cache,
topk_idx_2d.reshape(-1).to(torch.int32),
kv_cache.shape[1],
)
dequant_gather_slots(
workspace[n_topk_rows:],
swa_kv_cache,
swa_idx_2d.reshape(-1).to(torch.int32),
swa_kv_cache.shape[1],
)
# Use a shape-derived width so replay accepts changing lens without host sync.
_BLOCK_N = 16
width = max(_BLOCK_N, ((K_total + _BLOCK_N - 1) // _BLOCK_N) * _BLOCK_N)
pos = torch.arange(width, device=device, dtype=torch.int32).unsqueeze(0)
tok = torch.arange(num_tokens, device=device, dtype=torch.int32).unsqueeze(1)
slen = swa_lens.reshape(-1, 1).to(torch.int32).clamp(min=0, max=max_swa)
if use_topk and topk_lens is not None:
tlen = topk_lens.reshape(-1, 1).to(torch.int32).clamp(min=0, max=max_topk)
else:
tlen = torch.zeros_like(slen)
in_topk = pos < tlen
in_swa = (pos >= tlen) & (pos < tlen + slen)
topk_ws = tok * max_topk + pos
swa_ws = n_topk_rows + tok * max_swa + (pos - tlen)
combined_indices = torch.where(
in_topk,
topk_ws,
torch.where(in_swa, swa_ws, torch.full_like(topk_ws, -1)),
)
out_attn, _, _ = triton_bf16_mla_sparse_interface(
q=q,
kv=workspace.unsqueeze(1),
indices=combined_indices.unsqueeze(1),
sm_scale=softmax_scale,
d_v=q.shape[-1],
block_dpe=0,
)
out.copy_(out_attn)
-20
View File
@@ -1,20 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1 hardware-isolated model entry point."""
from vllm.platforms import current_platform
from .quant_config import DeepseekV4FP8Config
if current_platform.is_rocm():
from .amd.dspark import DSparkDeepseekV4ForCausalLM
from .amd.vl_model import DeepseekV41ForCausalLM
else:
from .nvidia.dspark import DSparkDeepseekV4ForCausalLM
from .nvidia.vl_model import DeepseekV41ForCausalLM
__all__ = [
"DSparkDeepseekV4ForCausalLM",
"DeepseekV4FP8Config",
"DeepseekV41ForCausalLM",
]
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-533
View File
@@ -1,533 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark draft model for DeepSeek-V4.1 (semi-autoregressive speculative decoding).
See: qwen3_dspark.py for base architecture. This one is specialized to the
DSV4.1 DSpark, which reuses the target model's architecture similarly to MTP.
Its weights ship in the target checkpoint under the ``mtp.{0,1,2}.*`` prefix.
To implement non-causal attention, we leverage the sparse attention implementation to
include the future query tokens in the top-k indices for each query token.
"""
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_linear_scale_param_name,
_use_sequence_parallel,
)
logger = init_logger(__name__)
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
class DSparkDeepseekV4Model(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
self.hidden_size = config.hidden_size
self.hc_mult = config.hc_mult
self.hc_eps = config.hc_eps
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = (
getattr(config, "n_mtp_layers", None)
or getattr(config, "num_nextn_predict_layers", None)
or 3
)
# Shared with the target (aliased by the speculator's loading utility).
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
self.main_proj = ReplicatedLinear(
config.hidden_size * len(self.target_layer_ids),
config.hidden_size,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "main_proj"),
)
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
)
current_vllm_config = get_current_vllm_config()
self.layers = nn.ModuleList(
[
DeepseekV4DecoderLayer(
current_vllm_config,
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
topk_indices_buffer=self.topk_indices_buffer,
)
for i in range(self.num_dspark_layers)
]
)
# Heads: final norm, and the Markov + confidence heads.
# Loaded from the "final" MTP layer weights (mtp.*) in the target
# checkpoint. v4.1 has no learned hc_head: the hc copies are
# collapsed with a pre-mix derived from the last layer's hc_ffn
# projection at the end of forward().
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", True):
self.confidence_head = DSparkConfidenceHead(
config.hidden_size + config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "confidence_head"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
"""
return self.main_norm(self.main_proj(aux_hidden_states))
@torch.inference_mode()
def precompute_and_store_context_kv(
self,
main_x: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
"""Insert the sliding-window context KV for every draft layer.
Mirrors the reference DSparkAttention: each layer derives its context KV
from the SAME projected target hidden ``main_x``, via that layer's own
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
layer's context slots.
``context_slot_mappings`` is a per-layer list (each entry is the context
slot mapping for that layer's kv-cache group, since the hybrid manager may
place draft layers in different groups). ``None`` (or a ``None`` entry)
runs the projection to reserve workspace but writes nothing (profiling).
"""
for i, layer in enumerate(self.layers):
slot_mapping = (
None if context_slot_mappings is None else context_slot_mappings[i]
)
attn = layer.attn
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
qr_kv, _ = attn.fused_wqa_wkv(main_x)
kv = qr_kv[..., attn.q_lora_rank :]
kv = attn.kv_norm(kv)
if slot_mapping is None:
continue
_insert_context_kv(attn, kv, context_positions, slot_mapping)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, inputs_embeds
)
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
residual = post_mix = res_mix = pre_mix = None
for layer in self.layers:
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
hidden_states,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
)
hidden_states = mhc_post_torch(hidden_states, residual, post_mix, res_mix)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
pre_mix = sp_all_gather(pre_mix)[:full_num_tokens]
# Collapse the hc copies with the pre-mix from the last layer's FFN
# mixes — the mix the reference forward_head applies (hc_pre with the
# last block's ffn pre-mix). Return the PRE-norm head hidden;
# compute_logits applies self.norm.
hidden_states = DeepseekV4DecoderLayer._hc_collapse(hidden_states, pre_mix)
return hidden_states
def _insert_context_kv(
attn: nn.Module,
kv: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
query and discard it, since context tokens have no query). Mirrors
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
"""
swa_cache = attn.swa_cache_layer.kv_cache
block_size = attn.swa_cache_layer.block_size
cos_sin_cache = attn.rotary_emb.cos_sin_cache
cache_dtype = swa_cache.dtype
n_ctx = kv.shape[0]
dummy_q = torch.zeros(
(n_ctx, attn.n_local_heads, attn.head_dim),
dtype=kv.dtype,
device=kv.device,
)
if cache_dtype == torch.uint8:
# fp8_ds_mla UE8M0 paged layout
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
dummy_q,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin_cache,
attn.padded_heads,
attn.eps,
block_size,
True, # apply_q_norm (unused: the query is a discarded dummy)
attn.kv_mxfp8,
)
elif cache_dtype == torch.bfloat16:
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
dummy_q,
kv,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn.eps,
block_size,
)
else: # per-tensor fp8 (torch.float8_e4m3fn)
# TODO(ben): double-check if this is being dispatched correctly for FI backend
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
dummy_q,
kv,
dummy_q_fp8,
swa_3d,
slot_mapping,
positions,
cos_sin_cache,
attn._flashinfer_fp8_kv_scale,
attn._flashinfer_fp8_q_scale_inv,
attn.eps,
block_size,
)
class DSparkDeepseekV4ForCausalLM(nn.Module):
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
# load_dspark_model always aliases the target's.
has_own_embed_tokens = False
has_own_lm_head = False
# Full-vocab draft: draft ids are target ids, no remapping needed.
draft_id_to_target_id = None
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
self.linear_scale_name = _linear_scale_param_name(
vllm_config, getattr(self.config, "expert_dtype", "fp4")
)
self.pad_shared_expert = getattr(
self.quant_config, "weight_block_size", None
) is not None and not _use_sequence_parallel(vllm_config)
self.model = DSparkDeepseekV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# Shared with the target (aliased by the speculator's load utility).
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
# --- Hooks used by the speculator -------------------------------------
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
return self.model.combine_hidden_states(aux_hidden_states)
def get_draft_kv_cache_layer_names(self) -> list[str]:
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
# layer, named by its prefix.
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
def precompute_and_store_context_kv(
self,
context_states: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mappings: list[torch.Tensor | None] | None = None,
) -> None:
self.model.precompute_and_store_context_kv(
context_states, context_positions, context_slot_mappings
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
# Returns the pre-norm collapsed head hidden ([T, hidden_size]).
return self.model(input_ids, positions, inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Base logits U_k = lm_head(norm(head_hidden))."""
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Full-vocab draft: base logits, no d2t scatter.
return self.compute_logits(hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
return draft_ids # full-vocab: draft ids are target ids
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
# --- Weight loading ----------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
Non-mtp weights (embed/head/main layers) belong to the target model and
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
# Draft MoE layers use the dspark_* expert counts, not the
# backbone's (see DeepseekV4MoE and the reference
# ModelArgs.get_moe_config).
n_draft_experts = (
getattr(self.config, "dspark_n_routed_experts", 0)
or self.config.n_routed_experts
)
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=n_draft_experts,
)
expert_scale_suffix = (
".weight_scale"
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
else ".weight_scale_inv"
)
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
stacked_params_mapping = [
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_confidence_head = False
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
continue
name = mapped
if "confidence_head." in name:
loaded_confidence_head = True
# ``.scale`` -> per-method scale suffix.
if name.endswith(".scale"):
suffix = (
expert_scale_suffix
if _EXPERT_SCALE_RE.search(name)
else f".{self.linear_scale_name}"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
)
# E8M0 expert scales: keep raw exponent bytes.
if ".experts." in name:
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
param = params_dict[name_mapped]
success = param.weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(name_mapped)
break
continue
# Stacked rules only apply to decoder-layer weights. Head-stack params
# (main_proj/norm/markov_head/confidence_head) load directly —
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
param.weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
break
else:
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
self.process_weights_after_loading()
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def process_weights_after_loading(self) -> None:
# ROCm linears and fused-MoE runners finalize their own quantized
# parameters. NVIDIA-only MegaMoE/WO-A requantization is not needed.
return
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
Returns None for non-mtp weights (owned by the target model).
"""
m = re.match(r"mtp\.(\d+)\.(.*)", name)
if m is None:
return None
stage = int(m.group(1))
rest = m.group(2)
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
return None
# The checkpoint calls the Markov head's factors ``embed``/``head``;
# DSparkMarkovHead registers them as ``markov_w1``/``markov_w2``.
if rest.startswith("markov_head.embed."):
return "model.markov_head.markov_w1." + rest.removeprefix(
"markov_head.embed."
)
if rest.startswith("markov_head.head."):
return "model.markov_head.markov_w2." + rest.removeprefix(
"markov_head.head."
)
# Head-stack params live at model level (mtp.last), context combiner at
# model level (mtp.0); everything else is a per-layer decoder block.
head_prefixes = (
"norm.",
"markov_head.",
"confidence_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
):
return f"model.{rest}"
return f"model.layers.{stage}.{rest}"
File diff suppressed because it is too large Load Diff
@@ -1,97 +0,0 @@
# DeepSeek-V4.1-Flash mono decode layer (gfx950, FlyDSL)
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
ATOM's V4.1 mono decode
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
from it, each file naming its origin; the attention stages (`attention/`) are
new, reading vLLM's KV records and weights as loaded.
## Scope
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
settled, without folding the previous output in, on layer 0 (the embedding
broadcast to the four streams) and after an Engram write (layers 1 and 14,
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
key table from that top-k. The FFN launch serves a layer outside these (an
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
The paths interleave freely (same tensors at the layer boundary).
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
routing bias is `e_score_correction_bias`).
- Contract: a drop-in for `DeepseekV4DecoderLayer.forward`:
in `x` (the previous FFN output, reduced, bf16 [M, 5120]), `residual`
(bf16 [M, 4, 5120]), `post_mix` [M, 4, 1], `res_mix` [M, 4, 4],
`pre_mix` [M, 4] (f32, the previous seam's)
out the MoE output (reduced), the residual after the FFN seam, and that
seam's post / comb / pre mixes.
## Launches
K1 seam (stages/seam.py): slice (160 x 32 columns): fold x into the
residual (post x + sum comb r), the collapsed input (pre . R'), partial
mixes (fn . R') and sums of squares -> gate (S): 24 mixes -> pre / post
/ comb (Sinkhorn 20) -> norm (S): attn_norm -> vLLM MXFP8 -> X8
attention front (attention/front.py): wqkv, q / kv norms + KV insert,
wq_b + RoPE -> Q
K2 attention back (attention/back.py): split attention, combine + inverse
RoPE, wo_a, wo_b; the bf16 wo_b partial pushed to every rank
FFN seam: reduce (the ranks' partials summed in rank order, fp32 ->
bf16: the attention output), slice, gate, norm -> the MoE input
MoE (stages/moe.py): xq, router (bf16 GEMV, f32 logits), route
(sqrtsoftplus + bias, top-6, renormalized x 1.5), shared expert (MXFP8
GEMVs, clamped SiLU), ug (MXFP4 x MXFP8 per expert of the step's
union), down (+ routing weight, top-k order; + shared) -> push, rank-order
sum -> out
FFN K2 without the attention back: push (160 x 32 columns) of vLLM's
unreduced wo_b output, as its bits, to every rank's attention region,
then K2's FFN seam and MoE (the same code, `layer.run_ffn`)
## Execution model and hand-offs
- Grid = 256 CTAs (one a CU, all resident), 512 threads a CTA, LDS a per-stage
union. A stage is a task table: CTA `b` runs tasks `b, b + 256, ...`, stages
in order; a consumer only waits on earlier stages, so no deadlock while the
grid is resident. A GEMV task issues its weight loads before it polls its
activations.
- In a launch: tagged pairs (value + tag in one 8 B store) or plain data +
flags, at device scope. The tag is the launch pair's epoch (K1 `2e + 1`,
K2 or the FFN launch `2e + 2`), a device word their CTA 0 moves on once every
CTA has marked the launch done: no clearing between steps, no host step hook, every argument
a fixed device pointer (graph-safe).
- Across ranks: one symmetric uncached buffer a rank (aiter `UncachedIpcHeap`),
every rank holding every peer's address; pushes are system-scope stores of
(value, tag) pairs. Every rank runs the same launches, so the epochs agree;
peer regions alternate by epoch parity, and a rank cannot run two steps ahead
of a peer (each step's reduce waits on every rank's push).
## Numerics (the vLLM path each stage reproduces)
- Seams: aiter `mhc_fused_post_pre_delayed_rmsnorm` (post fold order, fn as
bf16 hi + lo, products summed in order, Triton sigmoid / exp / rcp).
- Attention: vLLM's ROCm chain. `mxfp8(x)`: one E8M0 per 32 values,
`code = clamp(ceil(log2(amax / 448)) + 127, 0, 254)`; weights in vLLM's
row-major e4m3 [N, K] with [N/32, K/32] E8M0 scales (one shared copy); KV
records as vLLM's insert writes them (dims 0..447 e4m3 with a UE8M0 per 64,
RoPE dims bf16, 584 B); keys = the sliding window plus the top-k compressed
rows, softmax in base 2 with the attention sink folded in.
- MoE: vLLM's router (bf16 gate GEMV, f32 logits, `topk_hash_softplus_sqrt`)
and AITER a8w4 `fused_moe` (interleaved gate / up, swiglu limit 10, weights
in aiter's (16, 16) shuffle and `shuffle_scale` order); the shared expert as
vLLM's MXFP8 block-32 linears with `silu_and_mul_with_clamp`.
- All-reduces: exact, fp32 in rank order -> bf16. vLLM's decode all-reduces at
M <= 48 are below its INT4 quick-reduce threshold, so vLLM runs them exact too.
## Validation
- Per layer against vLLM's own modules in a TP process group (`DeepseekV4MoE`,
the mHC seams, the attention chain, the all-reduce), real checkpoint weights.
- Bench: layers of one type back to back in one HIP graph, cold weights,
us per layer, M = 1..48, TP2 / TP4; then e2e (AgentX, InferenceX#3690
recipe).
@@ -1,5 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1-Flash mono decode layer: one decoder layer of a decode step in
two persistent FlyDSL launches, its TP all-reduces in-kernel (README.md). The
host side is ``runner``; vLLM docks it in ``..mono_decode``."""
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The mono layer's attention stages: ``front`` (K1: wqkv, q / kv norms and the
KV insert, wq_b) and ``back`` (K2: split attention, combine, wo_a, wo_b)."""
@@ -1,888 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""K2 ``back``: one DeepSeek-V4.1 attention layer from the rotated query to the
rank's wo_b partial.
One launch a layer and rank, ``BLOCKS`` x ``THREADS``, S rows:
split (S x chunks of CK keys): flash-decoding over one chunk of a token's
keys (K1's key table: top-k compressed rows, then window rows) for
every head of the rank. The token's q in LDS; each wave one
(head tile, 16-key tile): its fp8_ds_mla rows loaded once,
dequantized into its QK operands and kept raw in LDS; QK (bf16
MFMA), the chunk's max / sum per head, bf16 p in LDS; PV over the
chunk's raw rows (wave w: dims 64 w ..) -> the chunk's partial
(bf16) and its (max, sum) per head, published
combine (S x H): a (token, head)'s chunks merged with the sink folded in,
bf16, inverse GPT-J RoPE of dims 448.., vLLM's MXFP8 of each 32
-> XO (published)
wo_a (G x 1024 / 32, two 16-row tiles): the group's XO rows times wo_a
-> bf16 -> MXFP8 of the 32 rows -> X8B (published)
wo_b (320, 16 rows): X8B times wo_b -> bf16 -> ``out``
Hand-offs are plain device-scope data behind tagged flags (half the bytes of a
tagged pair per value). Adapted from ATOM's V4.1 ``attn_post`` (K2a), which
reads a bf16 pool and ends in a peer all-reduce.
"""
from dataclasses import dataclass
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T, as_ir_value
from .device import (
BLOCKS,
CM_DEV,
THREADS,
WAVES,
Mailbox,
bf16_round,
bf_hi,
bf_lo,
butterfly,
clamp_fp8,
first_task,
fp8_pack4,
fp8x8_bf16,
fresh,
gload,
gstore,
hw_exp2,
lane_gather,
ld_f32,
ld_i32,
mfma_bf16,
mx_code,
mx_mul,
pow2,
rsrc,
spin_until,
stamp,
traced,
xshfl,
)
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
from .plan import (
DATA,
HALF,
HEAD_DIM,
HEAD_TILE,
KEYS,
NOPE,
ROPE,
ROWS,
WOA_ROWS,
Dims,
back_scratch,
cdiv,
front_end,
tile_rows,
token_tiles,
)
_STREAM = fx.Stream(None)
NEG = -3.4028234663852886e38
LOG2E = 1.4426950408889634
REC = DATA + 8 # a raw record row in LDS: 576 data bytes + 8 scale bytes
Q_ROW = (
HEAD_DIM // 2 + 4
) # a q row in LDS (i32 words): 512 bf16 + 16 B, no bank aliasing
P_PAD = 2 # i32 words past a P row: 16 head rows read together hit distinct banks
# wo_a's K order: waves 0..3 / 4..7 take the two 16-row tiles, a contiguous K
# range a wave, summed in wave order
WOA_SPLIT_K = WAVES * ROWS // WOA_ROWS
BACK_POINTS = 16 # start, stage ends, split internals, GEMV internals
@dataclass(frozen=True)
class BackBuild:
tokens: int
tp: int
ratio: int # 0: window only; 1 / 2: top-k over a compressed cache
timeline: bool = False
def plus(t0, v):
return v if isinstance(t0, int) and t0 == 0 else v + t0
def head_groups(d: Dims, s: int) -> int:
"""Wave groups a split unit's head tiles are spread over: two (a tile each,
64-key chunks, twice the units) while the units fit the grid, else one (a
wave's keys loaded and dequantized once for every head tile)."""
return 2 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= BLOCKS else 1
def chunk_keys(d: Dims, s: int) -> int:
"""A split unit's keys: a 16-key tile a wave of a head group."""
return 16 * (WAVES // head_groups(d, s))
def p_row(d: Dims, s: int) -> int:
"""A P row (one head's chunk of bf16 p) in LDS, i32 words."""
return chunk_keys(d, s) // 2 + P_PAD
@traced
def publish(c, region, i):
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if c["tid"] == 0:
c["mb"].put(region, i, fx.Int32(1))
@traced
def await_flags(c, region, n, index):
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
tid = c["tid"]
if tid < n:
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
gpu.barrier()
# ---------------------------------------------------------------- keys
def key_rec(c, t, k):
"""(data address, scale address, valid) of token t's key k from K1's key
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
none (it reads a zero record: its scores are masked, its values zeros)."""
a, ratio = c["args"], c["ratio"]
# a table this launch wrote (``krdy``) is read past the CU's cache
cm = CM_DEV if const_expr("krdy" in c) else 0
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
ok = val != fx.Int32(-1)
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
if const_expr(ratio > 0):
is_c = ok & (val < 0)
blk = is_c.select(a["comp_block"], a["swa_block"])
stride = is_c.select(a["comp_stride"], a["swa_stride"])
base = is_c.select(a["comp"], a["swa"])
else:
blk, stride, base = a["swa_block"], a["swa_stride"], a["swa"]
b = slot // blk
off = slot % blk
page = base + fx.Int64(b) * stride
z = a["zrec"]
data = ok.select(page + fx.Int64(off * DATA), z)
sc = ok.select(page + fx.Int64(blk * DATA + off * 8), z + fx.Int64(DATA))
return data, sc, ok
def _scale_byte(scw, blk64):
return (scw[blk64 // 4] >> (blk64 % 4 * 8)) & 0xFF
def _fp8_pair_f32(word, scale, hi):
v = rocdl.cvt_scalef32_pk_f32_fp8(
fx.Vector.make_type(2, fx.Float32),
as_ir_value(fx.Int32(word)),
as_ir_value(fx.Float32(scale)),
hi,
)
v = fx.Vector(v)
return [v[0], v[1]]
# ---------------------------------------------------------------- split
@traced
def stage_split(c, unit):
"""Flash-decoding over chunk ``unit % nchunk`` of token ``unit // nchunk``,
every head of the rank -> the chunk's partial and (max, sum) per head. Wave
(head group hg, key tile kt): keys 16 kt .. of the chunk (loaded and
dequantized once) against the group's head tiles; then dims 64 w .. of the
PV over every key and head."""
tid, lane, wave, d, s = c["tid"], c["lane"], c["wave"], c["d"], c["S"]
a, sl = c["args"], c["sl"]
nchunk = c["nchunk"]
t = unit // nchunk
ch = unit % nchunk
ck = chunk_keys(d, s)
ngrp = head_groups(d, s)
nkt = WAVES // ngrp # 16-key tiles a chunk
nht = d.head_tiles // ngrp # head tiles a wave
hgi = wave // nkt
kt = wave % nkt
g = lane // 16
r16 = lane % 16
k0 = ch * ck
if const_expr("krdy" in c):
await_flags(c, c["krdy"], 1, lambda u: t)
# the key count, this lane's key and q: independent loads, out together
# (the key table holds -1 past a token's keys)
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
key = k0 + kt * 16 + r16
data, sc, okv = key_rec(c, t, key)
qrow = d.heads * HEAD_DIM // 8
qv = []
for i in range_constexpr(cdiv(qrow, THREADS)):
u = fresh(tid) + THREADS * i
qv.append(
fx.Vector(
bo.buffer_load(
rsrc(a["q"]),
(t * d.q_rows) // 2 + fx.min(u, qrow - 1) * 4,
vec_width=4,
dtype=T.i32,
)
)
)
if k0 < kv_len:
# a key's 448 e4m3 as 7 x 16 B (lane g: dims 64 c + 16 g ..), its 64
# bf16 rope dims as 2 x 16 B (32 c + 8 g ..), its 7 scale bytes
scw = gload(sc, words=2)
nope = [
gload(data + fx.Int64(64 * c4 + 16 * g), words=4)
for c4 in range(NOPE // 64)
]
rope = [
gload(data + fx.Int64(NOPE + (cc * 32 + 8 * g) * 2), words=4)
for cc in range(ROPE // 32)
]
# q of every head into LDS (bf16 rows of 512 + pad), 16 B a thread
for i in range_constexpr(cdiv(qrow, THREADS)):
u = fresh(tid) + THREADS * i
if u < qrow:
fx.ptr_store(
qv[i],
sl["q"]
+ (u // (HEAD_DIM // 8)) * Q_ROW
+ (u % (HEAD_DIM // 8)) * 4,
)
# the raw rows to LDS for the PV (head group 0's waves)
if hgi == 0:
rawb = sl["raw"] + (kt * 16 + r16) * (REC // 4)
for c4 in range_constexpr(NOPE // 64):
fx.ptr_store(nope[c4], rawb + (64 * c4 + 16 * g) // 4)
for cc in range_constexpr(ROPE // 32):
fx.ptr_store(rope[cc], rawb + (NOPE + (cc * 32 + 8 * g) * 2) // 4)
if g == 0:
fx.ptr_store(scw, rawb + DATA // 4)
gpu.barrier()
stamp(c, 5)
# QK: A = the keys (dequantized once), B = q of a head tile (LDS); the
# contraction's 32-dim chunk 2 c + h of the nope dims is {64 c + 16 g +
# 8 h ..} (lane g's 16 B load), q read in the same order
scale = fx.Int32(a["qk_scale"]).bitcast(fx.Float32)
st = sl["st"]
accs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(nht)]
for c4 in range_constexpr(NOPE // 64):
ksc = pow2(_scale_byte(scw, c4))
for h2 in range_constexpr(2):
kw = fp8x8_bf16(nope[c4][2 * h2], nope[c4][2 * h2 + 1], ksc).bitcast(
fx.BFloat16
)
for ht in range_constexpr(nht):
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
qw = fx.Vector(
fx.ptr_load(
qb + 32 * c4 + 8 * g + 4 * h2,
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
for cc in range_constexpr(ROPE // 32):
kw = rope[cc].bitcast(fx.BFloat16)
for ht in range_constexpr(nht):
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
qw = fx.Vector(
fx.ptr_load(
qb + (NOPE + cc * 32 + 8 * g) // 2,
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
# scores of keys 4 g + i (C rows), head r16 of a tile (C column)
ok = (key < kv_len) & okv
okw = ok.select(fx.Int32(1), fx.Int32(0))
valid = [lane_gather(okw, 4 * g + i) != 0 for i in range(4)]
svs = []
for ht in range_constexpr(nht):
sv = [
fx.Float32(valid[i].select(accs[ht][i] * scale, fx.Float32(NEG)))
for i in range(4)
]
mw = butterfly(
fx.max(fx.max(sv[0], sv[1]), fx.max(sv[2], sv[3])), (32, 16), fx.max
)
if g == 0:
fx.ptr_store(mw, st + ((hgi * nht + ht) * nkt + kt) * HEAD_TILE + r16)
svs.append(sv)
gpu.barrier()
lst = st + d.heads * nkt
for ht in range_constexpr(nht):
hti = hgi * nht + ht
m = fx.ptr_load(st + (hti * nkt) * HEAD_TILE + r16)
for j in range_constexpr(1, nkt):
m = fx.max(m, fx.ptr_load(st + (hti * nkt + j) * HEAD_TILE + r16))
sv = svs[ht]
p = [
fx.Float32(valid[i].select(hw_exp2(sv[i] - m), fx.Float32(0.0)))
for i in range(4)
]
lw = (p[0] + p[2]) + (p[1] + p[3])
lw = lw + xshfl(lw, 32)
lw = lw + xshfl(lw, 16)
pw = (
fx.Vector.from_elements(p, fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)
)
pbase = (
sl["p"] + (hti * HEAD_TILE + r16) * p_row(d, s) + (kt * 16 + 4 * g) // 2
)
fx.ptr_store(fx.Vector.from_elements([pw[0], pw[1]], fx.Int32), pbase)
if g == 0:
fx.ptr_store(lw, lst + (hti * nkt + kt) * HEAD_TILE + r16)
gpu.barrier()
# chunk stats: (max, sum) per head
if tid < d.heads:
hts = tid // HEAD_TILE
hr = tid % HEAD_TILE
mm = fx.ptr_load(st + (hts * nkt) * HEAD_TILE + hr)
ll = fx.ptr_load(lst + (hts * nkt) * HEAD_TILE + hr)
for j in range_constexpr(1, nkt):
mm = fx.max(mm, fx.ptr_load(st + (hts * nkt + j) * HEAD_TILE + hr))
ll = ll + fx.ptr_load(lst + (hts * nkt + j) * HEAD_TILE + hr)
bo.buffer_store(
fx.Vector.from_elements([mm, ll], fx.Float32),
rsrc(c["pstat"]),
((t * nchunk + ch) * d.heads + tid) * 2,
cache_modifier=CM_DEV,
)
# PV: wave w dims 64 w + 4 r16 + dt (dt = 0..3: one LDS word holds the
# four of a key); the rope wave (dims 448..) reads bf16 pairs. Lane
# (r16, g) takes keys 4 g + j of a 16-key tile: A[dim][key]
is_rope = wave == WAVES - 1
dbase = 64 * wave
sh = (wave % 4) * 8
accs = [
[fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(4)]
for _ in range(d.head_tiles)
]
for kt2 in range_constexpr(nkt):
rb0 = sl["raw"] + (kt2 * 16 + 4 * g) * (REC // 4)
vv = fx.Vector.filled(16, 0.0, fx.Float32) # [dt][j]
if is_rope:
rv = []
for j in range_constexpr(4):
rw = fx.Vector(
fx.ptr_load(
rb0 + j * (REC // 4) + (NOPE + 8 * r16) // 4,
result_type=fx.Vector.make_type(2, fx.Int32),
)
)
rv.append([bf_lo(rw[0]), bf_hi(rw[0]), bf_lo(rw[1]), bf_hi(rw[1])])
vv = fx.Vector.from_elements(
[rv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
)
else:
nv = []
for j in range_constexpr(4):
rb = rb0 + j * (REC // 4)
w = fx.Int32(fx.ptr_load(rb + (dbase + 4 * r16) // 4))
scb = fx.Int32(fx.ptr_load(rb + DATA // 4 + wave // 4))
scl = pow2((scb >> sh) & 0xFF)
nv.append(
_fp8_pair_f32(w, scl, False) + _fp8_pair_f32(w, scl, True)
)
vv = fx.Vector.from_elements(
[nv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
)
for ht2 in range_constexpr(d.head_tiles):
pb = fx.Vector(
fx.ptr_load(
sl["p"]
+ (ht2 * HEAD_TILE + r16) * p_row(d, s)
+ (kt2 * 16 + 4 * g) // 2,
result_type=fx.Vector.make_type(2, fx.Int32),
)
).bitcast(fx.Int16)
for dt in range_constexpr(4):
av = (
fx.Vector.from_elements(
[vv[dt * 4 + j] for j in range(4)], fx.Float32
)
.to(fx.BFloat16)
.bitcast(fx.Int16)
)
accs[ht2][dt] = fx.Vector(
rocdl.mfma_f32_16x16x16bf16_1k(
T.vec(4, T.f32), [av, pb, accs[ht2][dt]]
)
)
stamp(c, 6)
# partial[t][ch][head][dim] bf16: lane (r16, g) holds C rows 4 g + i ->
# dims 64 w + 16 g + 4 i + dt of head ht2 * 16 + r16: 32 contiguous
# bytes, two 16 B stores (4 lanes fill a 128 B line)
for ht2 in range_constexpr(d.head_tiles):
hh = ht2 * HEAD_TILE + r16
words = []
for i in range_constexpr(4):
v4 = fx.Vector.from_elements(
[accs[ht2][dt][i] for dt in range(4)], fx.Float32
)
w2 = v4.to(fx.BFloat16).bitcast(fx.Int32)
words += [w2[0], w2[1]]
off = (((t * nchunk + ch) * d.heads + hh) * HEAD_DIM + dbase + 16 * g) // 2
for hlf in range_constexpr(2):
bo.buffer_store(
fx.Vector.from_elements(words[4 * hlf : 4 * hlf + 4], fx.Int32),
rsrc(c["part"]),
off + 4 * hlf,
cache_modifier=CM_DEV,
)
publish(c, c["srdy"], t * nchunk + ch)
stamp(c, 7)
# ---------------------------------------------------------------- combine
COMB_HEADS = WAVES # heads a combine unit: a wave each
@traced
def stage_combine(c, unit):
"""(token, 8 heads) ``unit``, a wave a head, 8 dims a lane: the chunks
merged with the sink folded in, bf16, inverse RoPE, MXFP8 of each 32 -> XO,
published (a flag a (token, head group))."""
lane, wave, d, s = c["lane"], c["wave"], c["d"], c["S"]
a = c["args"]
hg_n = d.heads // COMB_HEADS
t = unit // hg_n
hg = unit % hg_n
h = hg * COMB_HEADS + wave
nchunk = c["nchunk"]
ck = chunk_keys(d, s)
kv_len = ld_i32(a["klen"], t)
nlive = (kv_len + ck - 1) // ck
pos = ld_i32(a["pos"], 2 * t)
sink = ld_f32(a["sink"], h) * fx.Float32(LOG2E)
d0 = lane * 8 # this lane's 8 dims
rope_lane = d0 >= NOPE
jj = fx.max((d0 - NOPE) // 2, 0) # 4 pairs from here
cs = [
rope_lane.select(ld_f32(a["cos_sin"], pos * ROPE + jj + i), fx.Float32(1.0))
for i in range(4)
]
sn = [
rope_lane.select(
ld_f32(a["cos_sin"], pos * ROPE + HALF + jj + i), fx.Float32(0.0)
)
for i in range(4)
]
await_flags(c, c["srdy"], nlive, lambda u: t * nchunk + u)
m = fx.Float32(NEG)
ms, ls, vs = [], [], []
for cc in range_constexpr(nchunk):
live = cc < nlive
stv = fx.Vector(
bo.buffer_load(
rsrc(c["pstat"]),
((t * nchunk + cc) * d.heads + h) * 2,
vec_width=2,
dtype=T.f32,
)
)
pv = fx.Vector(
bo.buffer_load(
rsrc(c["part"]),
(((t * nchunk + cc) * d.heads + h) * HEAD_DIM + d0) // 2,
vec_width=4,
dtype=T.i32,
)
)
mc = fx.Float32(live.select(stv[0], fx.Float32(NEG)))
ms.append(mc)
ls.append(fx.Float32(live.select(stv[1], fx.Float32(0.0))))
vals = []
for e in range_constexpr(4):
vals += [bf_lo(pv[e]), bf_hi(pv[e])]
vs.append([fx.Float32(live.select(v, fx.Float32(0.0))) for v in vals])
m = fx.max(m, mc)
m_final = fx.max(m, sink)
lsum = hw_exp2(sink - m_final)
tot = [fx.Float32(0.0)] * 8
for cc in range_constexpr(nchunk):
wgt = hw_exp2(ms[cc] - m_final)
lsum = lsum + ls[cc] * wgt
tot = [tot[e] + vs[cc][e] * wgt for e in range(8)]
inv = fx.Float32(1.0) / lsum
o = [bf16_round(tot[e] * inv) for e in range(8)]
# inverse GPT-J RoPE of each pair: e' = e cos + o sin, o' = o cos - e sin
y = []
for i in range_constexpr(4):
e, od = o[2 * i], o[2 * i + 1]
y += [e * cs[i] + od * sn[i], od * cs[i] - e * sn[i]]
amax = abs(y[0])
for v in y[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max) # 32 dims = 4 lanes
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(y[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(y[4 + i] * mul) for i in range(4)])
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["xo"]),
t * (d.q_rows // 4) + (h * HEAD_DIM + d0) // 4,
cache_modifier=CM_DEV,
)
if lane % 4 == 0:
bo.buffer_store(
code,
rsrc(c["xos"]),
t * (d.q_rows // 32) + (h * HEAD_DIM + d0) // 32,
cache_modifier=CM_DEV,
)
publish(c, c["crdy"], t * hg_n + hg)
# ---------------------------------------------------------------- wo_a / wo_b
@traced
def stage_woa(c, task, ops, t0, n):
"""Rows 32 task .. of wo_a -> bf16 -> vLLM's MXFP8 of the 32 -> X8B."""
tid, red, d, lane = c["tid"], c["red"], c["d"], c["lane"]
split = WOA_SPLIT_K
gemv_mfmas(c, d.group_k, c["xl"], c["xsl"], red, ops, split, tiled=True, rows=n)
gpu.barrier()
mine = tid < WOA_ROWS * n
tl = fx.min(tid // WOA_ROWS, n - 1)
t = plus(t0, tl)
row = tid % WOA_ROWS
tile = row // ROWS
y = bf16_round(
row_sum(red, row % ROWS, tl, [tile * split + w for w in range(split)])
)
amax = butterfly(abs(y), (1, 2, 4, 8, 16), fx.max)
code = mx_code(amax)
q = clamp_fp8(y * mx_mul(code))
w = fp8_pack4(
q,
*[
lane_gather(q.bitcast(fx.Int32), fx.min(lane + k, 63)).bitcast(fx.Float32)
for k in (1, 2, 3)
],
)
if mine & (row % 4 == 0):
bo.buffer_store(
w,
rsrc(c["x8b"]),
t * (d.o_rows // 4) + (task * WOA_ROWS + row) // 4,
cache_modifier=CM_DEV,
)
if mine & (row == 0):
bo.buffer_store(
code, rsrc(c["x8bs"]), t * (d.o_rows // 32) + task, cache_modifier=CM_DEV
)
gpu.barrier()
def woa_loads(c, task):
a = c["args"]
return gemv_loads(
c,
a["woa"],
a["woa_s"],
c["d"].group_k,
task * WOA_ROWS,
WOA_SPLIT_K,
tiled=True,
)
def wob_loads(c, task, pred=None):
a = c["args"]
return gemv_loads(c, a["wob"], a["wob_s"], c["d"].o_rows, task * ROWS, pred=pred)
def _wob_store(a, t, col, v0, v1):
word = (
fx.Vector.from_elements([v0, v1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
bo.buffer_store(word, rsrc(a["out"]), (t * a["out_stride"] + col) // 2)
@traced
def stage_wob(c, task, ops, task2, ops2, has2, t0, n, mark=False):
"""Rows 16 task .. (and 16 task2 .., ``has2``) of wo_b -> bf16 -> out, or
(``c["wob_out"]``, the mono layer) the caller's hook with each bf16 pair."""
tid, red = c["tid"], c["red"]
o_rows = c["d"].o_rows
a = c["args"]
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red, ops, rows=n)
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red + RED, ops2, rows=n)
gpu.barrier()
if const_expr(mark):
stamp(c, 15)
per = ROWS * n // 2 # bf16 pairs a task
k = tid // per
emit = c.get("wob_out") or (lambda t, col, v0, v1: _wob_store(a, t, col, v0, v1))
if (tid < 2 * per) & ((k == 0) | has2):
u = tid % per
rp = u % (ROWS // 2)
tl = u // (ROWS // 2)
t = plus(t0, tl)
rk = red + k * RED
col = (k == 0).select(task, task2) * ROWS + 2 * rp
v0 = row_sum(rk, 2 * rp, tl)
v1 = row_sum(rk, 2 * rp + 1, tl)
emit(t, col, v0, v1)
gpu.barrier()
# ---------------------------------------------------------------- kernel
def back_plan(key):
"""(nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob) of a build."""
s, d = key.tokens, Dims(key.tp)
ck = chunk_keys(d, s)
nchunk = cdiv(KEYS, ck)
n_split = s * nchunk
n_comb = s * (d.heads // COMB_HEADS)
COMB0 = n_split % BLOCKS
assert d.woa_tasks <= BLOCKS
assert BLOCKS < d.wob_tasks <= 2 * BLOCKS
# the GEMVs past the attention units, so CTAs without attention work have
# their weights in flight from the start
WOA0 = (n_split + n_comb) % BLOCKS
WOB0 = (WOA0 + d.woa_tasks) % BLOCKS
# wo_b's weights before the wo_a stage only while the wo_a CTAs have no
# attention work (loads complete in issue order: else they would hold up
# wo_a's activation rows)
early_wob = n_split + n_comb + d.woa_tasks <= BLOCKS
return nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob
def back_lds_members(s: int, d: Dims) -> dict:
"""K2's attention LDS structs: the split stage's and the GEMVs' (a union's
members: one at a time)."""
rows = tile_rows(s)
ck = chunk_keys(d, s)
@fx.struct
class SplitLds:
q: fx.Array[fx.Int32, d.heads * Q_ROW, 16]
raw: fx.Array[fx.Int32, ck * REC // 4, 16]
p: fx.Array[fx.Int32, d.heads * p_row(d, s), 16]
st: fx.Array[fx.Float32, 2 * d.heads * (WAVES // head_groups(d, s)), 16]
@fx.struct
class GemvLds:
red: fx.Array[fx.Float32, 2 * RED, 16]
xl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 4), 16]
xsl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 32), 16]
return {"split": SplitLds, "gemv": GemvLds}
def back_lds(s: int, d: Dims):
"""K2's attention LDS: the union of ``back_lds_members``."""
m = back_lds_members(s, d)
SplitLds, GemvLds = m["split"], m["gemv"]
@fx.union
class BackLds:
split: SplitLds # type: ignore[valid-type]
gemv: GemvLds # type: ignore[valid-type]
return BackLds
def back_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
"""The attention back's context: LDS views (``lds``: an allocated
``back_lds``, or a dict of the members' views), mailbox (tag ``tag``),
scratch regions, arguments by name."""
s, d = key.tokens, Dims(key.tp)
tid = fx.thread_idx.x
if isinstance(lds, dict):
spl, gml = lds["split"], lds["gemv"]
else:
spl, gml = lds.split.peek(), lds.gemv.peek()
c = {
"S": s,
"tid": tid,
"bid": fx.block_idx.x,
"lane": tid % 64,
"wave": tid // 64,
"red": gml.red.ptr,
"xl": gml.xl.ptr,
"xsl": gml.xsl.ptr,
"d": d,
"ratio": key.ratio,
"mb": Mailbox(tag),
"nchunk": back_plan(key)[0],
"sl": {"q": spl.q.ptr, "raw": spl.raw.ptr, "p": spl.p.ptr, "st": spl.st.ptr},
"args": args,
}
for region, (off, _) in (
layout or back_scratch(s, d, start=front_end(s, d))
).items():
c[region] = scratch + fx.Int64(off)
return c
EPOCH_MARKS = 4 # word offset of the per-CTA marks in the epoch buffer
@traced
def epoch_begin(c, epoch, ep):
"""This CTA has read the launch pair's epoch: its own mark word (no
contended atomic) := ep + 1, which ``epoch_end`` checks."""
if c["tid"] == 0:
bo.buffer_store(
ep + 1, rsrc(epoch), EPOCH_MARKS + c["bid"], cache_modifier=CM_DEV
)
@traced
def epoch_end(c, epoch, ep, reset=None):
"""CTA 0, once every CTA marked this epoch (thread u checks CTA u's mark):
``reset()`` (state later pairs must find cleared), then the epoch moved on.
The next pair's hand-offs carry a tag no earlier pair wrote; later launches'
kernels start after this one ends."""
if c["bid"] == 0:
tid = c["tid"]
if tid < BLOCKS:
r = rsrc(epoch)
def load_mark():
return fx.Int32(
bo.buffer_load(
r,
EPOCH_MARKS + tid,
vec_width=1,
dtype=T.i32,
cache_modifier=CM_DEV,
)
)
_ = spin_until(load_mark, lambda v: v != ep + 1)
gpu.barrier()
if tid == 0:
if const_expr(reset is not None):
reset()
gstore(epoch, ep + 1, words=1)
@traced
def run_back(c, key, bid):
"""The attention back's stages on this CTA: split, combine, wo_a, wo_b."""
s = key.tokens
dd = Dims(key.tp)
# plain ints: a Python object read inside traced control flow is carried
woa_tasks, groups, heads, group_k = dd.woa_tasks, dd.groups, dd.heads, dd.group_k
q_rows, o_rows, wob_tasks = dd.q_rows, dd.o_rows, dd.wob_tasks
a = c["args"]
tiles = token_tiles(s)
_, n_split, n_comb, COMB0, WOA0, WOB0, early_wob = back_plan(key)
stamp(c, 0)
for unit in range(first_task(bid, 0), n_split, BLOCKS):
stage_split(c, unit)
stamp(c, 1)
for unit in range(first_task(bid, COMB0), n_comb, BLOCKS):
stage_combine(c, unit)
gpu.barrier()
stamp(c, 2)
# the GEMV weights, every CTA right after its attention work: its wo_a
# task (if any), then its one or two wo_b tasks
woa_task = first_task(bid, WOA0)
has_woa = woa_task < woa_tasks
woa_t = fx.min(woa_task, woa_tasks - 1)
ops = gemv_loads(
c,
a["woa"],
a["woa_s"],
group_k,
woa_t * WOA_ROWS,
WOA_SPLIT_K,
tiled=True,
pred=has_woa,
)
wob_task = first_task(bid, WOB0)
wob_second = wob_task + BLOCKS
has_second = wob_second < wob_tasks
def wob_weights():
return (
wob_loads(c, wob_task),
wob_loads(c, fx.min(wob_second, wob_tasks - 1), pred=has_second),
)
if const_expr(early_wob):
ops1, ops2 = wob_weights()
if const_expr(c.get("tl") is not None):
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
stamp(c, 8)
# wo_a: every token tile of its group's XO rows, the next tile's in flight
if has_woa:
grp = woa_t // (woa_tasks // groups)
await_flags(c, c["crdy"], s, lambda u: u * (heads // COMB_HEADS) + grp)
stamp(c, 9)
xo = (
c["xo"],
c["xos"],
q_rows // 4,
grp * group_k // 4,
group_k // 4,
q_rows // 32,
grp * group_k // 32,
group_k // 32,
)
regs = rows_loads(c, *xo, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, group_k // 4, group_k // 32, c["xl"], c["xsl"], n)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *xo, *tiles[ti + 1])
stage_woa(c, woa_t, ops, t0, n)
stamp(c, 10)
publish(c, c["ardy"], woa_t)
stamp(c, 3)
# wo_b
if const_expr(not early_wob):
ops1, ops2 = wob_weights()
stamp(c, 11)
await_flags(c, c["ardy"], woa_tasks, lambda u: u)
stamp(c, 12)
x8b = (
c["x8b"],
c["x8bs"],
o_rows // 4,
0,
o_rows // 4,
o_rows // 32,
0,
o_rows // 32,
)
regs = rows_loads(c, *x8b, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, o_rows // 4, o_rows // 32, c["xl"], c["xsl"], n)
if const_expr(ti == 0):
stamp(c, 14)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *x8b, *tiles[ti + 1])
stage_wob(
c,
wob_task,
ops1,
fx.min(wob_second, wob_tasks - 1),
ops2,
has_second,
t0,
n,
mark=ti == 0,
)
if const_expr(ti == 0):
stamp(c, 13)
stamp(c, 4)
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Device primitives of the DeepSeek-V4.1 attention mega kernel.
Wave reductions and shuffles, hardware math, bf16 / fp8 / MX packing, the
scaled MFMA, buffer and 64-bit global accesses, and the tagged-pair mailbox.
Adapted from ATOM's mono decode framework (``atom/mono/device``), itself ported
from the FlyDSL GLM-5 layer kernel.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
from aiter.ops.flydsl.kernels.kernels_common import kernel_signature
from flydsl._mlir import ir
from flydsl._mlir.dialects import llvm
from flydsl.compiler.ast_rewriter import ASTRewriter
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T, as_ir_value
# The execution model: one CTA a CU of an MI355X, all resident.
BLOCKS = 256
THREADS = 512
WAVES = THREADS // 64
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16, NT = 2. SC1 alone is
# device scope (past the per-XCD L2s); NT streams data read once.
CM_DEV = 16
CM_NT = 2
FP8_MAX = 448.0
FP8, FP4 = 0, 4 # f8f6f4 MFMA operand formats
UNIT_SCALE = 127 # E8M0 code of 1.0
FLT_MIN = 1.1754943508222875e-38
POLL_MAX = 12 # mailbox pairs polled a batch
def traced(fn):
"""FlyDSL's AST rewrite for a helper, so its ``if`` / ``while`` / ``for`` on
traced values lower to scf ops (kernels rewrite only their own body)."""
return ASTRewriter.transform(fn)
def kernel_symbol(stem, **params):
"""Kernel symbol that profilers demangle to ``aiter::<stem>_<params>``."""
name = f"{stem}_{kernel_signature(**params)}"
return f"_ZN5aiter{len(name)}{name}E"
def first_task(bid, base):
"""CTA ``bid``'s first task of a stage placed from CTA ``base``, then every
BLOCKS-th."""
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
# ----------------------------------------------------------------------------- memory
def rsrc(addr, nbytes=None):
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
def ld_i32(ptr, i, cm=0):
return fx.Int32(
bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.i32, cache_modifier=cm)
)
def ld_f32(ptr, i):
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
def ld_bf(ptr, i):
return fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
)
def ld_i64(ptr, i):
"""Element i of an i64 buffer (two dwords)."""
w = fx.Vector(bo.buffer_load(rsrc(ptr), i * 2, vec_width=2, dtype=T.i32))
return (fx.Int64(w[1]) << 32) | fx.Int64(fx.Uint32(w[0]))
def _gptr(addr):
return llvm.IntToPtrOp(
ir.Type.parse("!llvm.ptr<1>"), as_ir_value(fx.Int64(addr))
).result
def gload(addr, words=4, nt=False):
"""``words`` i32 (1, 2 or 4) at a 64-bit global address: a cache past 4 GiB is
reachable only this way (a buffer offset is 32 bits)."""
i32 = ir.IntegerType.get_signless(32)
ty = i32 if words == 1 else ir.VectorType.get([words], i32)
v = llvm.LoadOp(ty, _gptr(addr), alignment=4 * words, nontemporal=nt or None).result
return fx.Int32(v) if words == 1 else fx.Vector(v)
def atomic_add_dev(addr, value):
"""Device-scope atomic add on the global i32 at ``addr``; the old value.
Relaxed: an acquire / release here would write back and invalidate L2."""
return fx.Int32(
fx.atomic_add(
_gptr(addr),
fx.Int32(value),
syncscope="agent",
ordering=fx.AtomicOrdering.Monotonic,
)
)
def gstore(addr, value, words=4):
"""Store ``words`` i32 (an Int32 or a Vector) at a 64-bit global address."""
v = as_ir_value(fx.Int32(value) if words == 1 else value)
llvm.StoreOp(v, _gptr(addr), alignment=4 * words)
# ----------------------------------------------------------------------------- math
def fresh(v):
"""``v`` through an opaque move, so the lane math built on it is computed at
its use instead of being hoisted out of a task loop and held in registers."""
return fx.Int32(
llvm.inline_asm(
T.i32,
[fx.Int32(v).ir_value()],
"v_mov_b32 $0, $1",
"=v,v",
has_side_effects=True,
)
)
def tile_map(tid, width):
"""(tid // width, tid % width) for 0 <= tid < THREADS without a division:
shifts for a power of two, else a multiply-shift exact on that range."""
if width & (width - 1) == 0:
sh = width.bit_length() - 1
return tid >> sh, tid & (width - 1)
m = -(-(1 << 20) // width)
assert all((t * m) >> 20 == t // width for t in range(THREADS)), width
row = (tid * m) >> 20
return row, tid - row * width
def uniform(v):
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
def _hw_f32(name, x):
return fx.Float32(
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
)
def hw_rsq(x):
return _hw_f32("llvm.amdgcn.rsq.f32", x)
def hw_rcp(x):
return _hw_f32("llvm.amdgcn.rcp.f32", x)
def hw_exp2(x):
return _hw_f32("llvm.amdgcn.exp2.f32", x)
def fma(a, b, c):
return fx.Float32(fmath.fma(fx.Float32(a), fx.Float32(b), fx.Float32(c)))
def div_rn(x, d, r):
"""``x / d`` rounded to nearest (IEEE), given ``r = RN(1 / d)``."""
q = x * r
return fma(fma(-q, d, x), r, q)
def permlane_swap(off, x, y):
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
pr = swap(
llvm.StructType.get_literal([T.i32, T.i32]),
as_ir_value(fx.Int32(x)),
as_ir_value(fx.Int32(y)),
False,
False,
)
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
def lane_gather(v, src_lane):
"""Lane ``src_lane``'s value of v (ds_bpermute)."""
return fx.Int32(
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
)
def lane_f32(v, src_lane):
return lane_gather(v.bitcast(fx.Int32), src_lane).bitcast(fx.Float32)
def readlane(v, src_lane):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
r = fx.Int32(rocdl.readlane(T.i32, bits, fx.Int32(src_lane)))
return r.bitcast(fx.Float32) if isinstance(v, fx.Float32) else r
def xshfl(v, off):
"""Value of lane ``lane ^ off`` (DPP in a row, permlane swaps across)."""
assert off in (1, 2, 4, 8, 16, 32), off
if off >= 16:
return v.shuffle_xor(off, 64)
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
if off == 8:
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
elif off == 4:
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
elif off == 2:
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
else:
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
return y.bitcast(fx.Float32) if is_f else y
def xred(v, off, op):
if off < 16:
return op(v, xshfl(v, off))
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
a, b = permlane_swap(off, x, x)
if is_f:
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
return op(type(v)(a), type(v)(b))
def _add(a, b):
return a + b
def butterfly(v, offsets, op=_add):
"""Butterfly reduction over the xor ``offsets``, in that order."""
for off in offsets:
v = xred(v, off, op)
return v
def wave_sum(v):
return butterfly(v, (32, 16, 8, 4, 2, 1))
def wave_max(v):
return butterfly(v, (32, 16, 8, 4, 2, 1), fx.max)
def row_shr(v, lane, k):
"""Lane - k of the same 16-lane row, else 0 (DPP row_shr)."""
moved = lane_gather(v.bitcast(fx.Int32), fx.max(lane - k, 0)).bitcast(fx.Float32)
return fx.Float32((lane % 16 >= k).select(moved, fx.Float32(0.0)))
# ----------------------------------------------------------------------------- packing
def bf16_round(a):
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
def bf16_pair(a, b):
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
return (
fx.Vector.from_elements([a, b], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Float32)[0]
)
def bf_lo(w):
return (w << 16).bitcast(fx.Float32)
def bf_hi(w):
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
def fp8_pack4(a, b, c, d):
"""Four f32 (in range) -> one dword of four E4M3 bytes, RNE."""
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
def fp8x8_bf16(w0, w1, scale):
"""Eight E4M3 of two dwords times an f32 ``scale`` -> eight bf16 as a
4-dword Vector (exact for a power-of-two scale in range)."""
out = []
for w in (w0, w1):
for hi in (False, True):
p = rocdl.cvt_scalef32_pk_bf16_fp8(
T.bf16x2, as_ir_value(fx.Int32(w)), as_ir_value(fx.Float32(scale)), hi
)
out.append(fx.Vector(p).bitcast(fx.Int32)[0])
return fx.Vector.from_elements(out, fx.Int32)
def clamp_fp8(v):
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
def pow2(code):
"""The fp32 power of two a biased exponent (1..254) stands for."""
return (code << 23).bitcast(fx.Float32)
def rcp_pow2(code):
"""1 / ``pow2(code)``, exact for code in [1, 253]."""
return ((254 - code) << 23).bitcast(fx.Float32)
def ceil_exp(v):
"""Biased exponent of the smallest power of two >= v (v > 0 finite, normal)."""
bits = v.bitcast(fx.Int32)
return ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
fx.Int32(1), fx.Int32(0)
)
def mx_code(amax):
"""VLLM's ROCm MXFP8 scale code of a 32-group: clamp(ceil(log2(amax / 448))
+ 127, 0, 254), amax floored at FLT_MIN."""
a = fx.max(amax, fx.Float32(FLT_MIN))
q = div_rn(a, fx.Float32(FP8_MAX), fx.Float32(1.0 / FP8_MAX))
return fx.min(fx.max(ceil_exp(q), fx.Int32(0)), fx.Int32(254))
def mx_mul(code):
"""2^(127 - code) as fp32, for code in [0, 254] (the quant multiplier)."""
return ((254 - code) << 23).bitcast(fx.Float32)
# ----------------------------------------------------------------------------- MFMA
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8):
"""16x16x128 scaled MFMA (gfx950 f8f6f4): lane l (row l % 16) holds K bytes
16 (l // 16) .. +16 and 64 + 16 (l // 16) .. +16 of the 128-step, and the
E8M0 of K block l // 16."""
return fx.Vector(
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, 0, sa, 0, sb]
)
)
def mfma_bf16(a, b, c):
"""16x16x32 bf16 MFMA: lane l (row / col l % 16) holds K 8 (l // 16) .. +8."""
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
# ----------------------------------------------------------------------------- mailbox
@traced
def _spin(load_all, pending):
v = load_all()
while pending(v):
rocdl.s_sleep(1)
v = load_all()
return v
def spin_until(load, pending):
"""``load()`` again (s_sleep between) while ``pending(value)``; the value."""
return _spin(load, pending)
class Mailbox:
"""Tagged-pair hand-off between the CTAs of one launch (device scope).
Every 32-bit value is stored next to the launch's tag in one 8-byte store; a
consumer polls until every tag matches, so a hand-off is one round trip with
no separate flag. The tag is ``epoch * 64 + layer + 1`` (``epoch`` a device
counter bumped once a step): a pair from an earlier layer or step never
carries a live tag, and nothing is cleared between launches."""
def __init__(self, tag):
self.tag = tag
def put(self, base, i, v):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
bo.buffer_store(
fx.Vector.from_elements([bits, self.tag], fx.Int32),
rsrc(base),
i * 2,
cache_modifier=CM_DEV,
)
def put_bf(self, base, i, vs):
"""Elements i .. i + len(vs) (2 or 4, i aligned) as bf16 pairs, one store."""
words = []
for j in range_constexpr(len(vs) // 2):
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(base),
i,
cache_modifier=CM_DEV,
)
def put_words(self, base, i, words):
"""Raw 32-bit words (1 or 2) at pairs i.. (fp8 words must come this way)."""
vals = []
for w in words:
vals += [fx.Int32(w), self.tag]
bo.buffer_store(
fx.Vector.from_elements(vals, fx.Int32),
rsrc(base),
i * 2,
cache_modifier=CM_DEV,
)
def poll(self, specs, batch=POLL_MAX):
"""Batched poll of ``specs`` = [(base, pair index, npairs in {1, 2})]: all
re-loaded while any tag is stale. Returns the value words."""
if const_expr(len(specs) == 0):
return []
if const_expr(len(specs) > batch):
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
tag = self.tag
def load_all():
words = []
for b, i, n in specs:
w = fx.Vector(
bo.buffer_load(
rsrc(b),
fx.Int32(i) * 2,
vec_width=2 * n,
dtype=T.i32,
cache_modifier=CM_DEV,
)
)
words += [w[e] for e in range(2 * n)]
return fx.Vector.from_elements(words, fx.Int32)
nw = sum(2 * n for _, _, n in specs)
def pending(v):
bad = v[1] != tag
for e in range_constexpr(3, nw, 2):
bad = bad | (v[e] != tag)
return bad
v = _spin(load_all, pending)
outs, e = [], 0
for _, _, n in specs:
outs.append([v[e + 2 * q] for q in range(n)])
e += 2 * n
return outs
LOAD_BATCH = 4
@traced
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
"""A CTA's strided loop over v = tid + THREADS i < n: ``batch`` rounds' loads
issued before their uses; a round past n loads element n - 1, uses nothing."""
rounds = -(-n // THREADS)
for b0 in range_constexpr(0, rounds, batch):
idx = range(b0, min(b0 + batch, rounds))
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
for i, g in zip(idx, got):
v = tid + THREADS * i
if v < n:
use(v, g)
def barrier():
gpu.barrier()
# ----------------------------------------------------------------------------- timeline
def memrealtime():
"""s_memrealtime: the 100 MHz device clock."""
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
@traced
def stamp(c, point):
"""A timeline build's stamp: thread 0 of the CTA stores the clock at
``point`` (``c["tl"]`` [BLOCKS, points] i64; None: not a timeline build)."""
if const_expr(c.get("tl") is None):
return
if c["tid"] == 0:
t = memrealtime()
w = fx.Vector.from_elements([fx.Int32(t), fx.Int32(t >> 32)], fx.Int32)
bo.buffer_store(w, rsrc(c["tl"]), (c["bid"] * c["tl_points"] + point) * 2)
@@ -1,680 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""K1 ``front``: one DeepSeek-V4.1 attention layer from its normed input row to
the rotated query and the step's KV rows in the sliding-window cache.
One launch a layer and rank, ``BLOCKS`` CTAs x ``THREADS`` threads, S rows.
Every CTA issues its weight loads (its wqkv part and its wq_b tasks) at the
kernel start, before it waits on anything; the stages:
xq (S > 16: S x 2 K halves, on the 32 CTAs without a wqkv part): the
bf16 input's half -> vLLM's MXFP8 -> X8 (published, a flag each)
kt (S): the token's attention keys for K2 -- top-k compressed slots
through the kv source's block table, then window slots -> KT
wqkv_a (112 row tiles x 2 K halves, 16 rows): the input's MXFP8 half --
quantized in the CTA (S <= 16) or X8's -- a token tile at a time
(the next tile's rows in flight) times wqkv's -> fp32 partials ->
QKVP
qkv (S): the halves summed -> bf16; q_norm -> bf16 -> MXFP8 -> QX8
(published, a flag a token); kv_norm -> bf16 -> GPT-J RoPE -> bf16
-> the fp8_ds_mla record at the token's slot
wq_b (H x 512 / 16, 16 rows): QX8 times wq_b -> bf16 -> GPT-J RoPE of
each head's dims 448.. -> q (global, bf16)
Each elementwise step follows vLLM's ROCm path (``mxfp8_quantize``,
``FusedQKVRMSNorm``, ``fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert``);
the GEMV reductions differ from the Triton GEMMs' only in their order.
Adapted from ATOM's V4.1 ``attn_pre`` (K1), less its mHC seam.
"""
from dataclasses import dataclass
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T
from .device import (
BLOCKS,
CM_DEV,
THREADS,
Mailbox,
bf16_round,
bf_hi,
bf_lo,
butterfly,
ceil_exp,
clamp_fp8,
div_rn,
first_task,
fp8_pack4,
fresh,
gstore,
hw_rsq,
ld_f32,
ld_i32,
mx_code,
mx_mul,
rsrc,
stamp,
tile_map,
traced,
xshfl,
)
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
from .plan import (
DATA,
EPS,
HALF,
HEAD_DIM,
HIDDEN,
KEYS,
KV_DIM,
NOPE,
Q_RANK,
QKV_ROWS,
ROPE,
ROWS,
TILE,
TOPK,
WINDOW,
WQKV_KPARTS,
WQKV_TASKS,
Dims,
front_scratch,
tile_rows,
token_tiles,
)
_STREAM = fx.Stream(None)
X_WORDS = HIDDEN // 4
X_GROUPS = HIDDEN // 32
QX_WORDS = Q_RANK // 4
KSTEPS = HIDDEN // 128 // WQKV_KPARTS # a wqkv part's 128-steps
KWORDS = KSTEPS * 32
KGROUPS = KSTEPS * 4
XH = HIDDEN // WQKV_KPARTS # a K half's values
XH_COLS = XH // 8 // 2 # 16 B items (8 values) a half of a K half
XH_RPR = THREADS // XH_COLS # half-rows a round
XQ_ITEMS = XH // 8 # an xq unit's 16 B items (one a thread)
FRONT_POINTS = 6 # start, kt done, wqkv done, qkv done, wq_b done, (spare)
@dataclass(frozen=True)
class FrontBuild:
tokens: int
tp: int
ratio: int # the layer's compress ratio (0: no top-k keys)
timeline: bool = False
def plus(t0, v):
return v if isinstance(t0, int) and t0 == 0 else v + t0
@traced
def publish(c, region, i):
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if c["tid"] == 0:
c["mb"].put(region, i, fx.Int32(1))
@traced
def await_flags(c, region, n, index):
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
tid = c["tid"]
if tid < n:
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
gpu.barrier()
# ---------------------------------------------------------------- xq
def xq_loads(c, unit, pred):
"""(token, K half) ``unit``'s bf16 input: its load only, 8 values a thread
(threads past the half mirror its last item)."""
a = c["args"]
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
col = fx.min(c["tid"], XQ_ITEMS - 1)
e = (unit >> 1) * a["x_stride"] + (unit & 1) * XH + col * 8
return fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32))
@traced
def stage_xq(c, unit, w, live):
"""``xq_loads``' values -> vLLM's MXFP8 -> X8 words / X8S codes of the
unit (published by the caller)."""
tid, lane = c["tid"], c["lane"]
vals = []
for d in range_constexpr(4):
vals += [bf_lo(w[d]), bf_hi(w[d])]
amax = abs(vals[0])
for v in vals[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max)
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
t = unit >> 1
part = unit & 1
if live & (tid < XQ_ITEMS):
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["x8"]),
t * X_WORDS + part * KWORDS + tid * 2,
cache_modifier=CM_DEV,
)
if lane % 4 == 0:
bo.buffer_store(
code,
rsrc(c["x8s"]),
t * X_GROUPS + part * KGROUPS + (tid >> 2),
cache_modifier=CM_DEV,
)
# ---------------------------------------------------------------- kt
@traced
def stage_kt(c, t):
"""Token t's keys for K2: ``KT[t][k] = slot | 1 << 31`` for its top-k compressed
rows (k < ntopk), the window slot for k in [ntopk, kv_len), -1 for an
invalid or absent key; KLEN[t] = kv_len (0 for a pad row), KLEN[S + t] =
ntopk."""
tid = c["tid"]
a, ratio = c["args"], c["ratio"]
# device scope where this launch reads the table (K2's)
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
slot = ld_i32(a["slot"], 2 * t)
live = slot >= 0
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
if const_expr(ratio > 0):
pos = ld_i32(a["pos"], 2 * t)
ntopk = live.select(fx.min((pos + 1) // ratio, fx.Int32(TOPK)), fx.Int32(0))
req = ld_i32(a["t2r"], t)
else:
ntopk = fx.Int32(0)
kv_len = ntopk + nswa
for i in range_constexpr((KEYS + THREADS - 1) // THREADS):
k = fresh(tid) + THREADS * i
sidx = fx.min(fx.max(k - ntopk, 0), WINDOW - 1)
s_slot = ld_i32(a["swa_idx"], t * WINDOW + sidx)
val = (k < kv_len).select(s_slot, fx.Int32(-1))
if const_expr(ratio > 0):
local = ld_i32(a["topk"], t * TOPK + fx.min(k, TOPK - 1))
lc = fx.max(local, 0)
e = a["comp_block"]
blk = ld_i32(a["comp_bt"], req * a["bt_stride"] + lc // e)
c_slot = (blk * e + lc % e) | fx.Int32(-2147483648)
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
val = (k < ntopk).select(c_val, val)
if k < KEYS:
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
if tid == 0:
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
# ---------------------------------------------------------------- wqkv_a
def x_half_loads(c, part, pred, t0, n):
"""Tokens t0 .. t0 + n's bf16 input, K half ``part``: its loads only (16 B a
lane-item, 8 values; a thread one column of [2 n half-rows, 160]), masked
off on CTAs without a wqkv part."""
a = c["args"]
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
hr0, col = tile_map(c["tid"], XH_COLS)
e0 = part * XH + col * 8
out = []
for i in range((2 * n + XH_RPR - 1) // XH_RPR):
hr = fx.min(hr0 + XH_RPR * i, 2 * n - 1)
e = (t0 + (hr >> 1)) * a["x_stride"] + (hr & 1) * (XH // 2) + e0
out.append(fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32)))
return out
@traced
def quant_x_half(c, regs, n):
"""``x_half_loads``' values -> vLLM's MXFP8 rows of the K half in LDS (a
32-group = 4 lanes)."""
tid, lane = c["tid"], c["lane"]
x_row, s_row = lds_row(KWORDS), lds_row(KGROUPS)
hr0, col = tile_map(tid, XH_COLS)
act = tid < XH_RPR * XH_COLS
for i in range_constexpr(len(regs)):
hr = hr0 + XH_RPR * i
w = regs[i]
vals = []
for d in range_constexpr(4):
vals += [bf_lo(w[d]), bf_hi(w[d])]
amax = abs(vals[0])
for v in vals[1:]:
amax = fx.max(amax, abs(v))
amax = butterfly(amax, (1, 2), fx.max)
code = mx_code(amax)
mul = mx_mul(code)
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
tok = hr >> 1
half = hr & 1
if act & (hr < 2 * n):
fx.ptr_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
c["xl"] + (tok * x_row + half * (KWORDS // 2) + col * 2),
)
if lane % 4 == 0:
fx.ptr_store(
code, c["xsl"] + (tok * s_row + half * (KGROUPS // 2) + (col >> 2))
)
gpu.barrier()
@traced
def stage_wqkv(c, task, part, ops, t0, n, live):
"""Rows 16 task .. of wqkv over K half ``part`` against tokens t0 .. t0 + n
-> fp32 partials QKVP[part] (``live``: a CTA with a wqkv part)."""
tid, red = c["tid"], c["red"]
gemv_mfmas(c, HIDDEN, c["xl"], c["xsl"], red, ops, rows=n, nsteps=KSTEPS)
gpu.barrier()
if (tid < ROWS * n) & live:
r = tid % ROWS
tl = tid // ROWS
t = plus(t0, tl)
v = row_sum(red, r, tl)
c["mb"].put(c["qkvp"], (part * c["S"] + t) * QKV_ROWS + task * ROWS + r, v)
gpu.barrier()
# ---------------------------------------------------------------- q / kv norms
@traced
def stage_qkv(c, t):
"""Token t: the K halves summed -> bf16; q_norm -> bf16 -> MXFP8 (QX8);
kv_norm -> bf16 -> RoPE -> bf16 -> the fp8_ds_mla record at
slot_mapping[t]. 256 threads (8 q columns and 2 kv columns each); threads
256.. mirror thread 255 and store nothing."""
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
a, s = c["args"], c["S"]
tt = fx.min(tid, 255)
mine = tid < 256
qc = [fx.min(tt * 8 + j, Q_RANK - 1) for j in range(8)]
# independent loads first: they overlap the partials' wait
qw = [c["ldbf"](a["qn_w"], qc[j]) for j in range(8)]
kw = [c["ldbf"](a["kvn_w"], tt * 2 + j) for j in range(2)]
pos = ld_i32(a["pos"], 2 * t)
slot = ld_i32(a["slot"], 2 * t)
jr = tt - NOPE // 2
is_rope = jr >= 0
jj = fx.max(jr, 0)
cs = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + jj), fx.Float32(1.0))
sn = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + HALF + jj), fx.Float32(0.0))
specs = []
for p in range_constexpr(2):
specs += [(c["qkvp"], (p * s + t) * QKV_ROWS + qc[j], 1) for j in range(8)]
specs += [
(c["qkvp"], (p * s + t) * QKV_ROWS + Q_RANK + tt * 2 + j, 1)
for j in range(2)
]
got = c["mb"].poll(specs)
vals = [
bf16_round(got[j][0].bitcast(fx.Float32) + got[10 + j][0].bitcast(fx.Float32))
for j in range(10)
]
xq = [((tt * 8 + j) < Q_RANK).select(vals[j], fx.Float32(0.0)) for j in range(8)]
xk = vals[8:10]
ssq = xq[0] * xq[0]
for j in range_constexpr(1, 8):
ssq = ssq + xq[j] * xq[j]
ssq = butterfly(ssq, (32, 16, 8, 4, 2, 1))
ssk = butterfly(xk[0] * xk[0] + xk[1] * xk[1], (32, 16, 8, 4, 2, 1))
if (lane == 0) & mine:
fx.ptr_store(ssq, red + wave)
fx.ptr_store(ssk, red + 4 + wave)
gpu.barrier()
tq = (fx.ptr_load(red + 0) + fx.ptr_load(red + 1)) + (
fx.ptr_load(red + 2) + fx.ptr_load(red + 3)
)
tk = (fx.ptr_load(red + 4) + fx.ptr_load(red + 5)) + (
fx.ptr_load(red + 6) + fx.ptr_load(red + 7)
)
rq = hw_rsq(
div_rn(tq, fx.Float32(float(Q_RANK)), fx.Float32(1.0 / Q_RANK))
+ fx.Float32(EPS)
)
rk = hw_rsq(
div_rn(tk, fx.Float32(float(KV_DIM)), fx.Float32(1.0 / KV_DIM))
+ fx.Float32(EPS)
)
# q: bf16 normed row, then vLLM's MXFP8 (a 32-group = 4 threads)
ys = [bf16_round((xq[j] * rq) * qw[j]) for j in range(8)]
amax = abs(ys[0])
for y in ys[1:]:
amax = fx.max(amax, abs(y))
code = mx_code(butterfly(amax, (1, 2), fx.max))
mul = mx_mul(code)
if const_expr(c.get("index", False)): # noqa: SIM102
# the bf16 qr, vLLM's indexer's input
if mine & (tt * 8 < Q_RANK):
words = [
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
for i in range(4)
]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a["qr_out"]),
t * (Q_RANK // 2) + 4 * tt,
)
if mine & (tt * 8 < Q_RANK):
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["qx8"]),
t * QX_WORDS + 2 * tt,
cache_modifier=CM_DEV,
)
if tt % 4 == 0:
bo.buffer_store(
code,
rsrc(c["qx8s"]),
t * (Q_RANK // 32) + tt // 4,
cache_modifier=CM_DEV,
)
# kv: this thread's pair (2 tt, 2 tt + 1)
kn = [bf16_round((xk[j] * rk) * kw[j]) for j in range(2)]
e0 = bf16_round(kn[0] * cs - kn[1] * sn)
e1 = bf16_round(kn[0] * sn + kn[1] * cs)
# record quant of the nope dims: a 64-block = 32 threads (half a wave)
amax = butterfly(fx.max(abs(e0), abs(e1)), (1, 2, 4, 8, 16), fx.max)
rcode = ceil_exp(
div_rn(
fx.max(amax, fx.Float32(1e-4)), fx.Float32(448.0), fx.Float32(1.0 / 448.0)
)
)
inv = mx_mul(rcode)
q0 = clamp_fp8(e0 * inv)
q1 = clamp_fp8(e1 * inv)
n0 = xshfl(q0, 1)
n1 = xshfl(q1, 1)
word = fp8_pack4(q0, q1, n0, n1)
rope_word = (
fx.Vector.from_elements([e0, e1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
blk = a["swa_block"]
sblock = slot // blk
soff = slot % blk
base = a["swa"] + fx.Int64(sblock) * fx.Int64(a["swa_stride"])
rec = base + fx.Int64(soff * DATA)
if mine & (tt % 32 == 0) & (tt < NOPE // 2):
fx.ptr_store(rcode.bitcast(fx.Float32), red + 8 + tt // 32)
gpu.barrier()
if mine & (slot >= 0):
if tt < NOPE // 2:
if tt % 2 == 0:
gstore(rec + fx.Int64(2 * tt), word, words=1)
else:
gstore(rec + fx.Int64(NOPE + 4 * (tt - NOPE // 2)), rope_word, words=1)
if tt == 0:
sb = [fx.ptr_load(red + 8 + b).bitcast(fx.Int32) for b in range(7)]
w0 = sb[0] | (sb[1] << 8) | (sb[2] << 16) | (sb[3] << 24)
w1 = sb[4] | (sb[5] << 8) | (sb[6] << 16)
sc = base + fx.Int64(blk * DATA + soff * 8)
gstore(sc, fx.Vector.from_elements([w0, w1], fx.Int32), words=2)
publish(c, c["qrdy"], t)
# ---------------------------------------------------------------- wq_b
def wqb_rope(c, task0, pos_t, n):
"""(cos, sin) of this thread's pair in ``stage_wqb``'s epilogue over a tile
of n rows (``pos_t``: its token's position; every task of the CTA shares
the pair's dims: tasks BLOCKS apart are whole heads apart)."""
a = c["args"]
rp = c["tid"] % (ROWS // 2)
dim = (task0 * ROWS) % HEAD_DIM + 2 * rp
jj = fx.max((dim - NOPE) >> 1, 0)
return ld_f32(a["cos_sin"], pos_t * ROPE + jj), ld_f32(
a["cos_sin"], pos_t * ROPE + HALF + jj
)
def wqb_pos(c, t0, n):
"""This thread's token position for ``stage_wqb``'s epilogue of tile
t0 .. t0 + n."""
per = ROWS * n // 2
tl = fx.min((c["tid"] % per) // (ROWS // 2), n - 1)
return ld_i32(c["args"]["pos"], 2 * (t0 + tl))
@traced
def stage_wqb(c, task0, bops, t0, n, rope):
"""Rows 16 (task0 + BLOCKS k) .. of the rank's H x 512 query rows, every
task k of the CTA: GEMVs -> bf16 -> GPT-J RoPE of a head's dims 448..
(fp32) -> bf16 -> q."""
tid, red, d = c["tid"], c["red"], c["d"]
a = c["args"]
nt = len(bops)
for k in range_constexpr(nt):
gemv_mfmas(c, Q_RANK, c["qxl"], c["qxsl"], red + k * RED, bops[k], rows=n)
gpu.barrier()
per = ROWS * n // 2 # bf16 pairs a task
if tid < nt * per:
k = tid // per
u = tid % per
rp = u % (ROWS // 2)
tl = u // (ROWS // 2)
t = plus(t0, tl)
r0 = (task0 + BLOCKS * k) * ROWS + 2 * rp
rk = red + k * RED
v0 = bf16_round(row_sum(rk, 2 * rp, tl))
v1 = bf16_round(row_sum(rk, 2 * rp + 1, tl))
is_rope = (task0 * ROWS) % HEAD_DIM + 2 * rp >= NOPE
cs, sn = rope
e0 = is_rope.select(v0 * cs - v1 * sn, v0)
e1 = is_rope.select(v0 * sn + v1 * cs, v1)
word = (
fx.Vector.from_elements([e0, e1], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
)
bo.buffer_store(word, rsrc(a["q"]), (t * d.q_rows + r0) // 2)
gpu.barrier()
# ---------------------------------------------------------------- kernel
def front_lds(s: int, d: Dims):
"""K1's LDS (the attention front's stages)."""
rows = tile_rows(s)
wqb_per_cta = d.wqb_tasks // BLOCKS
@fx.struct
class FrontLds:
red: fx.Array[fx.Float32, wqb_per_cta * RED + 16, 16]
qxl: fx.Array[fx.Int32, rows * lds_row(QX_WORDS), 16]
qxsl: fx.Array[fx.Int32, rows * lds_row(Q_RANK // 32), 16]
xl: fx.Array[fx.Int32, rows * lds_row(KWORDS), 16]
xsl: fx.Array[fx.Int32, rows * lds_row(KGROUPS), 16]
return FrontLds
def front_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
"""The front stages' context: LDS views, mailbox (tag ``tag``), scratch
regions, arguments (``args``: the kernel's, by name)."""
s, d = key.tokens, Dims(key.tp)
tid = fx.thread_idx.x
c = {
"S": s,
"tid": tid,
"bid": fx.block_idx.x,
"lane": tid % 64,
"wave": tid // 64,
"red": lds.red.ptr,
"xl": lds.xl.ptr,
"xsl": lds.xsl.ptr,
"qxl": lds.qxl.ptr,
"qxsl": lds.qxsl.ptr,
"mb": Mailbox(tag),
"d": d,
"ratio": key.ratio,
"ldbf": lambda ptr, i: fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
),
"args": args,
}
for region, (off, _) in (layout or front_scratch(s, d)).items():
c[region] = scratch + fx.Int64(off)
return c
@traced
def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
"""The front's stages on this CTA. ``x8_given``: a stage before this one
(the mono layer's seam norm) publishes X8 / XRDY for every token (no input
quantization here). ``before_wqkv()`` runs ahead of the wqkv weights' loads
(a latency-critical stage of the caller's), ``after_wqkv()`` right after."""
s = key.tokens
wqb_tasks = Dims(key.tp).wqb_tasks
a = c["args"]
tid = c["tid"]
tiles = token_tiles(s)
wqb_per_cta = wqb_tasks // BLOCKS
assert wqb_tasks % BLOCKS == 0
n_wqkv = WQKV_TASKS * WQKV_KPARTS # 224
assert n_wqkv <= BLOCKS
n_free = BLOCKS - n_wqkv # CTAs without a wqkv part: xq, kt
QKV0 = 0 # on wqkv CTAs
WQB0 = 0
in_cta_quant = s <= TILE and not x8_given
xq_stage = s > TILE and not x8_given
xq_units = WQKV_KPARTS * s
xq_per = (xq_units + n_free - 1) // n_free
stamp(c, 0)
# ---- Loads complete in issue order, so every CTA issues its latency-
# critical loads before its later stages' weights: a wqkv CTA its first
# input tile and its wqkv part, the next tiles while it quantizes, then
# its wq_b tasks; a CTA with a key-table task those after that work.
# The q / kv norms run on wqkv CTAs, their partials' wait behind the
# wq_b weights that are due by then anyway.
wqkv_unit = first_task(bid, 0)
has_wqkv = wqkv_unit < n_wqkv
task = fx.min(wqkv_unit, n_wqkv - 1) % WQKV_TASKS
part = fx.min(wqkv_unit, n_wqkv - 1) // WQKV_TASKS
wqb_task = first_task(bid, WQB0)
free = bid - n_wqkv
is_free = free >= 0
if const_expr(in_cta_quant):
xregs = [x_half_loads(c, part, has_wqkv, *tiles[0])]
if const_expr(xq_stage):
xunits = [free + n_free * i for i in range(xq_per)]
xlive = [is_free & (u < xq_units) for u in xunits]
xq_regs = [
xq_loads(c, fx.min(fx.max(u, 0), xq_units - 1), lv)
for u, lv in zip(xunits, xlive)
]
if const_expr(before_wqkv is not None):
before_wqkv()
kops = gemv_loads(
c,
a["wqkv"],
a["wqkv_s"],
HIDDEN,
task * ROWS,
kbeg=part * KSTEPS,
nsteps=KSTEPS,
pred=has_wqkv,
)
if const_expr(after_wqkv is not None):
after_wqkv()
if const_expr(xq_stage): # noqa: SIM102 (a build-time and a run-time test)
if is_free:
for i in range_constexpr(xq_per):
stage_xq(
c, fx.min(fx.max(xunits[i], 0), xq_units - 1), xq_regs[i], xlive[i]
)
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if tid < xq_per:
u = free + n_free * tid
if u < xq_units:
c["mb"].put(c["xrdy"], u, fx.Int32(1))
if const_expr(not c.get("index", False)):
# an index source's top-k comes after this launch: K2 builds its table
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
stage_kt(c, t)
stamp(c, 1)
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
# ---- wqkv_a: one (row tile, K half) a CTA, every token tile; the wq_b
# weights issued behind the input rows' loads
def wqb_weights():
return [
gemv_loads(c, a["wqb"], a["wqb_s"], Q_RANK, (wqb_task + BLOCKS * i) * ROWS)
for i in range(wqb_per_cta)
]
if const_expr(in_cta_quant):
if has_wqkv:
quant_x_half(c, xregs[0], tiles[0][1])
bops = wqb_weights()
if has_wqkv:
stage_wqkv(c, task, part, kops, *tiles[0], has_wqkv)
else:
await_flags(c, c["xrdy"], s, lambda u: WQKV_KPARTS * u + part)
xr = (
c["x8"],
c["x8s"],
X_WORDS,
part * KWORDS,
KWORDS,
X_GROUPS,
part * KGROUPS,
KGROUPS,
)
regs = rows_loads(c, *xr, tiles[0][0], tiles[0][1], pred=has_wqkv)
if const_expr(len(tiles) == 1):
bops = wqb_weights()
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, KWORDS, KGROUPS, c["xl"], c["xsl"], n)
if const_expr(ti + 1 < len(tiles)):
t1, n1 = tiles[ti + 1]
regs = rows_loads(c, *xr, t1, n1, pred=has_wqkv)
if const_expr(ti + 2 == len(tiles)):
bops = wqb_weights()
stage_wqkv(c, task, part, kops, t0, n, has_wqkv)
stamp(c, 2)
# ---- q / kv norms, KV insert
for t in range(first_task(bid, QKV0), s, BLOCKS):
stage_qkv(c, t)
stamp(c, 3)
# ---- wq_b: every task's rows of a token tile at a time, the next
# tile's rows in flight
ropes = [wqb_rope(c, wqb_task, poss[ti], tiles[ti][1]) for ti in range(len(tiles))]
await_flags(c, c["qrdy"], s, lambda u: u)
qx = (c["qx8"], c["qx8s"], QX_WORDS, 0, QX_WORDS, Q_RANK // 32, 0, Q_RANK // 32)
regs = rows_loads(c, *qx, *tiles[0])
for ti in range_constexpr(len(tiles)):
t0, n = tiles[ti]
rows_store(c, regs, QX_WORDS, Q_RANK // 32, c["qxl"], c["qxsl"], n)
if const_expr(ti + 1 < len(tiles)):
regs = rows_loads(c, *qx, *tiles[ti + 1])
stage_wqb(c, wqb_task, bops, t0, n, ropes[ti])
stamp(c, 4)
@@ -1,234 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 GEMV of the mega kernel: 16 rows of a row-major e4m3 weight [N, K]
(vLLM's loaded layout, E8M0 scales in 32x32 blocks [N / 32, K / 32]) against
every token of a tile, its MXFP8 rows in LDS, on the gfx950 scaled MFMA.
A task may cover a window of K (``kbeg`` .. ``kbeg + nsteps`` 128-steps): a
split-K part, its activations in LDS holding just that window. The weight
operands are loaded first (all in flight, before the task waits on its
activations); wave w takes the window's steps w, w + 8, ... (or, ``split``, a
contiguous range of waves 0 .. split - 1), its 16 x 16 partial in ``red``,
summed in wave order by ``row_sum``. Adapted from ATOM's
``attn_pre.gemv_fp8_*``, which reads aiter's (16, 16)-preshuffled layout.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr
from flydsl.expr.typing import T
from .device import THREADS, WAVES, mfma_scaled, rsrc, tile_map, traced
# a token's LDS row of MXFP8 words / E8M0 codes, padded by 16 B so 16 rows read
# by one MFMA operand do not share a bank group
LDS_PAD = 4
RED = WAVES * 64 * 4 # one task's accumulators in ``red`` (f32)
def lds_row(words):
return words + LDS_PAD
def lds_at(u, row):
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
return u + u // row * LDS_PAD
def _steps(c, nsteps, split, tiled):
"""This wave's steps of the window: [(local step, live)]."""
wave = c["wave"]
if const_expr(split is None):
per = (nsteps + WAVES - 1) // WAVES
ragged = nsteps % WAVES != 0
return [
(wave + WAVES * i, (wave + WAVES * i < nsteps) if ragged else None)
for i in range(per)
]
assert split <= WAVES and nsteps % split == 0
per = nsteps // split
part = wave % split if tiled else fx.min(wave, split - 1)
return [(part * per + i, None) for i in range(per)]
def _step_load(c, r_w, r_ws, k, row0, st, pred=None):
"""Global step ``st``'s weight operand of rows row0 .. +16: two 16 B chunks a
lane (K bytes 16 j .. and 64 + 16 j .. of the step, row row0 + lane % 16)
and the E8M0 byte of its K block. ``pred`` false: masked (out of range: no
memory traffic, zeros)."""
lane = c["lane"]
j = lane // 16
row = row0 + lane % 16
base = row * k + 128 * st + 16 * j
lo = fx.Vector(bo.buffer_load(r_w, base // 4, vec_width=4, dtype=T.i32))
hi = fx.Vector(bo.buffer_load(r_w, (base + 64) // 4, vec_width=4, dtype=T.i32))
kg = k // 32
wsi = (row0 // 32) * kg + st * 4 + j
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
def gemv_loads(
c, w, ws, k, row0, split=None, tiled=False, kbeg=0, nsteps=None, pred=None
):
"""The weight operands of rows row0 .. +16 (``tiled``: row0 + 16 (wave //
split)) over steps kbeg .. kbeg + nsteps for this wave, all in flight
(``pred`` false: masked off, no memory traffic)."""
nsteps = k // 128 if nsteps is None else nsteps
if const_expr(pred is None):
r_w, r_ws = rsrc(w), rsrc(ws)
else:
# a masked-off CTA reads through a zero-size buffer: zeros, no traffic
nb = pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
r_w, r_ws = rsrc(w, nb), rsrc(ws, nb)
if const_expr(tiled):
row0 = row0 + 16 * (c["wave"] // split)
return [
_step_load(
c,
r_w,
r_ws,
k,
row0,
kbeg + (st if live is None else fx.min(st, nsteps - 1)),
pred,
)
for st, live in _steps(c, nsteps, split, tiled)
]
def _step_mfma(c, nsteps, xl, xsl, st, ops, acc, live, rows):
lane = c["lane"]
lo, hi, sa = ops
x_row, s_row = lds_row(nsteps * 32), lds_row(nsteps * 4)
j = lane // 16
col = fx.min(lane % 16, rows - 1)
if const_expr(live is not None):
st = fx.min(st, nsteps - 1)
av = fx.Vector.from_elements(
[lo[d] for d in range(4)] + [hi[d] for d in range(4)], fx.Int32
)
xlo = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xhi = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + 16 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xv = fx.Vector.from_elements(
[xlo[d] for d in range(4)] + [xhi[d] for d in range(4)], fx.Int32
)
sb = fx.ptr_load(xsl + (col * s_row + st * 4 + j))
if const_expr(live is not None):
# a step past the end adds zero at unit scale
av = fx.Vector.from_elements(
[live.select(av[d], fx.Int32(0)) for d in range(8)], fx.Int32
)
sa = live.select(sa, fx.Int32(127))
sb = live.select(sb, fx.Int32(127))
return mfma_scaled(av, xv, acc, sa, sb)
def gemv_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=16, nsteps=None):
"""``gemv_loads``' operands against ``rows`` tokens' MXFP8 rows of the
window in LDS (``xl`` words, ``xsl`` codes, rows ``lds_row`` of the window)
-> ``red[wave][lane]`` (a 16 x 16 C: row 4 (lane // 16) + i, token lane % 16)."""
lane, wave = c["lane"], c["wave"]
nsteps = k // 128 if nsteps is None else nsteps
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i, (st, live) in enumerate(_steps(c, nsteps, split, tiled)):
acc = _step_mfma(c, nsteps, xl, xsl, st, ops[i], acc, live, rows)
if const_expr(split is not None and not tiled):
acc = fx.Vector.from_elements(
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
fx.Float32,
)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
def row_sum(red, r, t, waves=None):
"""Row r, token t of the waves' accumulators, summed in wave order."""
tot = fx.Float32(0.0)
for w in waves if waves is not None else range(WAVES):
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
return tot
def span_items(width, n):
"""A [n, width] span of 16 B items: rows a round of THREADS threads, rounds."""
wq = width // 4
rpr = THREADS // wq
assert rpr >= 1, width
return wq, rpr, (n + rpr - 1) // rpr
def rows_loads(
c,
words_base,
codes_base,
stride_w,
off_w,
words,
stride_g,
off_g,
groups,
t0,
n,
pred=None,
):
"""Tokens t0 .. t0 + n's published MXFP8 row slices -- ``words`` i32 from
``off_w`` and ``groups`` codes (an i32 each) from ``off_g``, rows
``stride_w`` / ``stride_g`` i32 apart -- loaded (issued, not waited on),
16 B a lane-item: a thread one column, rows a round apart. Plain loads:
their flags are seen, and nothing in the launch read these lines before.
``pred`` false: masked off (zeros, no memory traffic)."""
tid = c["tid"]
nb = None if pred is None else pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
out = []
for base, stride, off, width in (
(words_base, stride_w, off_w, words),
(codes_base, stride_g, off_g, groups),
):
wq, rpr, rounds = span_items(width, n)
row0, col = tile_map(tid, wq)
a0 = col * 4 + off + t0 * stride
r = rsrc(base, nb)
regs = []
for i in range(rounds):
row = row0 + rpr * i
if (i + 1) * rpr > n or rpr * wq < THREADS:
row = fx.min(row, n - 1)
regs.append(
fx.Vector(
bo.buffer_load(r, row * stride + a0, vec_width=4, dtype=T.i32)
)
)
out.append(regs)
return out
@traced
def _store_span(c, regs, width, dst, n):
tid = c["tid"]
wq, rpr, _ = span_items(width, n)
row0, col = tile_map(tid, wq)
lrow = lds_row(width)
base = dst + (row0 * lrow + col * 4)
for i in range_constexpr(len(regs)):
if const_expr((i + 1) * rpr <= n and rpr * wq == THREADS):
fx.ptr_store(regs[i], base + i * rpr * lrow)
else:
if (row0 + rpr * i < n) & (tid < rpr * wq):
fx.ptr_store(regs[i], base + i * rpr * lrow)
def rows_store(c, regs, words, groups, xl, xsl, n):
"""``rows_loads``' registers into LDS rows 0 .. n (``lds_row`` strides)."""
_store_span(c, regs[0], words, xl, n)
_store_span(c, regs[1], groups, xsl, n)
gpu.barrier()
@@ -1,173 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Geometry of the DeepSeek-V4.1 attention mega kernel: model widths, a TP
rank's share of them, task counts and the scratch (mailbox) layout."""
from dataclasses import dataclass
HIDDEN = 5120
Q_RANK = 1280
KV_DIM = 512
QKV_ROWS = Q_RANK + KV_DIM # 1792
HEAD_DIM = 512
NOPE = 448
ROPE = 64
HALF = ROPE // 2
N_HEADS = 64
O_GROUPS = 8
O_RANK = 1024
WINDOW = 128
TOPK = 512
EPS = 1e-20
ROWS = 16 # GEMV rows a task
TILE = 16 # tokens a GEMV pass (an MFMA's N)
HEAD_TILE = 16 # heads an attention MFMA takes
WOA_ROWS = 32 # a wo_a task's rows: one MX group of its output
MAX_TOKENS = 48
# KV record (V4 fp8_ds_mla): 576 B data rows (448 e4m3 | 64 bf16), then 8 B of
# scales a token (7 UE8M0 + pad) after the block's rows
DATA = 576
SCALE_BYTES = 8
# attention: a split is 16 keys; a token has at most TOPK + WINDOW keys
BK = 16
SPLITS = 64 # split slots a (token, head tile): 1024 keys >= TOPK + WINDOW
COLS = 32 # PV columns a group (one MX group of the output)
GROUPS = HEAD_DIM // COLS # 16
PAIR = 8 # a mailbox (value, tag) pair
def cdiv(a, b):
return -(-a // b)
def round_up(a, b):
return cdiv(a, b) * b
@dataclass(frozen=True)
class Dims:
tp: int
def __post_init__(self):
assert self.tp in (2, 4), self.tp
@property
def heads(self) -> int:
return N_HEADS // self.tp
@property
def head_tiles(self) -> int:
return self.heads // HEAD_TILE
@property
def groups(self) -> int:
return O_GROUPS // self.tp
@property
def group_k(self) -> int:
"""wo_a's K a group: its eight heads' outputs."""
return self.heads // self.groups * HEAD_DIM # 4096
@property
def o_rows(self) -> int:
return self.groups * O_RANK
@property
def q_rows(self) -> int:
return self.heads * HEAD_DIM
@property
def wqb_tasks(self) -> int:
return self.q_rows // ROWS
@property
def woa_tasks(self) -> int:
return self.o_rows // WOA_ROWS
@property
def wob_tasks(self) -> int:
return HIDDEN // ROWS # 320
WQKV_TASKS = QKV_ROWS // ROWS # 112
WQKV_KPARTS = 2 # wqkv's K split: 224 (row tile, K half) parts
KEYS = TOPK + WINDOW # a token's attention keys at most
def token_tiles(s):
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
def tile_rows(s):
return min(s, TILE)
def pair_layout(items, start=0, align=256):
out, off = {}, start
for name, pairs in items:
off = round_up(off, align)
out[name] = (off, pairs * PAIR)
off += pairs * PAIR
return out
def byte_layout(items, start=0, align=256):
out, off = {}, start
for name, nbytes in items:
off = round_up(off, align)
out[name] = (off, nbytes)
off += nbytes
return out
def front_scratch(s: int, d: Dims) -> dict:
"""K1's scratch regions -> (byte offset, bytes): published data (plain
words behind a flag) and mailbox pairs."""
return byte_layout(
[
("x8", s * HIDDEN), # e4m3 words (S > 16: quantized once, shared)
("x8s", s * HIDDEN // 32 * 4), # an i32 code a group
("xrdy", WQKV_KPARTS * s * PAIR), # a flag a (token, K half)
("qkvp", WQKV_KPARTS * s * QKV_ROWS * PAIR),
("qx8", s * Q_RANK), # e4m3 words
("qx8s", s * Q_RANK // 32 * 4), # an i32 code a group
("qrdy", s * PAIR),
]
)
def back_scratch(s: int, d: Dims, start: int = 0) -> dict:
"""K2's scratch regions, laid out past ``start``: published data (plain)
and flag pairs."""
ck = (
64 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= 256 else 128
) # back.chunk_keys
nchunk = cdiv(KEYS, ck)
return byte_layout(
[
("part", s * nchunk * d.heads * HEAD_DIM * 2), # bf16 partials
("pstat", s * nchunk * d.heads * 8), # (max, sum) f32
("srdy", s * nchunk * PAIR),
("xo", s * d.q_rows), # e4m3 words
("xos", s * d.q_rows // 32 * 4), # an i32 code a group
("crdy", s * d.heads // 8 * PAIR),
("x8b", s * d.o_rows),
("x8bs", s * d.o_rows // 32 * 4),
("ardy", d.woa_tasks * PAIR),
],
start=start,
)
def front_end(s: int, d: Dims) -> int:
"""Where K2's regions start: past K1's, which share the launch tag."""
return round_up(max(o + n for o, n in front_scratch(s, d).values()), 256)
def scratch_bytes(s: int, d: Dims) -> int:
b = back_scratch(s, d, start=front_end(s, d))
return round_up(max(o + n for o, n in b.values()), 4096)
@@ -1,9 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/__init__.py
"""The mono kernels' shared mechanisms: ``ops`` (wave / math primitives, the TP
group's peer buffers), ``sync`` (the tagged mailbox), ``mx`` (FP8 / MX
arithmetic), ``plan`` (execution model, layouts, build keys) and
``peer_memory`` (symmetric peer buffers)."""
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/mx.py
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
tile's operand loads and the scaled MFMA that takes them.
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
FP8_MAX = 448.0
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
FP8, FP4 = 0, 4
UNIT_SCALE = 127 # the E8M0 code of 1.0
def clamp_fp8(v):
"""``v`` clamped to the E4M3 range."""
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
def code_ceil(v):
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
clamped to [1, 254]: the codes a scale may take."""
bits = v.bitcast(fx.Int32)
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
fx.Int32(1), fx.Int32(0)
)
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
def rcp_pow2(code):
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
bit for bit, a multiply where the division is a dozen instructions."""
return ((254 - code) << 23).bitcast(fx.Float32)
def scale_index(row, col, cols):
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
r32, a, b = row // 32, (row // 16) % 2, row % 16
c8, d, e = col // 8, (col // 4) % 2, col % 4
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
def fp4_tile_load(r_w, base, rg, k, st, at):
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
Nontemporal: a weight a step reads once."""
return fx.Vector(
bo.buffer_load(
r_w,
base + rg * (2 * k) + st * 256 + at,
vec_width=4,
dtype=T.i32,
cache_modifier=CM_NT,
)
)
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
byte ``sb_byte``)."""
return fx.Vector(
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
)
)
@@ -1,300 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/ops.py
# atom/mono/device/ranks.py
# atom/mono/device/stamps.py
"""Device-side primitives of the mono kernels: wave reductions and shuffles, the
approximate hardware math, bf16 / fp8 packing, buffer resources, the
execution-model constants and the kernel symbol naming.
Ported from the FlyDSL GLM-5 layer kernel (ROCm/FlyDSL branch
``glm5-mla-moe-layer-monokernel`` @ 1e28fd8a,
``kernels/mla_moe_layer/shared_reuse_moe_kernel.py``). Only published flydsl APIs
and aiter's vendored ``buffer_ops`` / ``dpp_utils`` are used.
Every function but ``kernel_symbol`` is traced inside a ``@flyc.kernel`` body.
Raw boundaries kept, for want of an equivalent on the pinned FlyDSL:
``buffer_ops`` loads / stores (the kernels take raw Int64 addresses, no tensors
for ``make_buffer_tensor``); raw MFMA intrinsics (hand-laid operands: permlane
transposes, fp8 packing, MXFP4 scales); the LLVM intrinsics here (``expect``,
``s_memrealtime``, the approximate rsq / rcp / exp2) and the struct unpacking of
``permlane*_swap``.
Also the TP group seen from inside a kernel (every rank's peer buffer, an
all-reduce's rank-ordered sum) and a timeline build's phase stamps."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
from flydsl._mlir.dialects import llvm
from flydsl.compiler.ast_rewriter import ASTRewriter
from flydsl.expr import const_expr, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T, as_ir_value
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES
# ---------------------------------------------------------------- primitives
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16. SC1 alone is device
# scope (past the per-XCD caches); SC0 | SC1 is system scope (peers over XGMI).
CM_DEV = 16
CM_SYS = 17
# NT = 2: nontemporal. A weight a layer reads once streams past the caches it
# would only evict the step's reused data from (the MoE at 48 rows: 143 -> 133 us)
CM_NT = 2
def rsrc(addr, nbytes=None):
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
def permlane_swap(off, x, y):
"""v_permlane{32,16}_swap on registers (x, y): off 32 trades x's upper-half
lanes with y's lower-half lanes; off 16 trades x's odd 16-lane rows with y's
even rows (each within its half). -> (x', y')."""
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
pr = swap(
llvm.StructType.get_literal([T.i32, T.i32]),
as_ir_value(fx.Int32(x)),
as_ir_value(fx.Int32(y)),
False,
False,
)
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
def lane_gather(v, src_lane):
"""Lane ``src_lane``'s value of v, a source lane a lane (ds_bpermute)."""
return fx.Int32(
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
)
def memrealtime():
"""s_memrealtime: the 100 MHz device clock (no FlyDSL wrapper)."""
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
def fresh(v):
"""``v`` (a per-lane Int32) through an opaque move, here: the lane math built
on it is computed at its use, where LLVM would otherwise hoist it out of a task
loop or CSE it across stages into registers held for the whole kernel (spills,
and a kernel with scratch is refused: ``runtime.widths``)."""
return fx.Int32(
llvm.inline_asm(
T.i32,
[fx.Int32(v).ir_value()],
"v_mov_b32 $0, $1",
"=v,v",
has_side_effects=True,
)
)
def uniform(v):
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
def _hw_f32(name, x):
return fx.Float32(
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
)
def hw_rsq(x):
"""v_rsq_f32 (~1 ulp, no libm range fixups)."""
return _hw_f32("llvm.amdgcn.rsq.f32", x)
def hw_rcp(x):
return _hw_f32("llvm.amdgcn.rcp.f32", x)
def hw_exp2(x):
return _hw_f32("llvm.amdgcn.exp2.f32", x)
def div_rn(x, d, r):
"""``x / d`` rounded to nearest, bit-identical to the IEEE division, given
``r = 1.0 / d`` from one true division per divisor (Markstein: a quotient
within 1 ulp plus its exact fma residual times RN(1/d) rounds correctly)."""
q = x * r
return fx.Float32(fmath.fma(fx.Float32(fmath.fma(-q, d, x)), r, q))
def xshfl(v, off):
"""Value of lane ``lane ^ off``: offsets 32 / 16 via v_permlane*_swap, in-row
offsets via DPP (VALU latency, not LDS). Powers of two only: any other
offset would take the lane ^ 1 path."""
assert off in (1, 2, 4, 8, 16, 32), off
if off >= 16:
return v.shuffle_xor(off, 64)
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
if off == 8:
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
elif off == 4:
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
elif off == 2:
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
else:
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
return y.bitcast(fx.Float32) if is_f else y
def xred(v, off, op):
"""op(v, lane ^ off) for a symmetric op; 32 / 16 take both halves of one swap."""
if off < 16:
return op(v, xshfl(v, off))
is_f = isinstance(v, fx.Float32)
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
a, b = permlane_swap(off, x, x)
if is_f:
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
return op(type(v)(a), type(v)(b))
def _add(a, b):
return a + b
def butterfly(v, offsets, op=_add):
"""Butterfly reduction over the given xor offsets, in the given order.
The order is the numerical contract: the aiter per-head norm reduces 32
lanes with offsets 16, 8, 4, 2, 1, and matching it keeps rstd bit-exact."""
for off in offsets:
v = xred(v, off, op)
return v
def traced(fn):
"""Apply flydsl's AST rewriting to a helper, so its ``while`` / ``if`` over
traced values lower to scf ops. Kernels rewrite only their own body."""
return ASTRewriter.transform(fn)
LOAD_BATCH = 4 # rounds of a strided loop whose loads go out together
@traced
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
"""A CTA's strided loop over elements v = tid + THREADS i < n (n a Python
int): ``load(v)`` of ``batch`` rounds issued before any ``use(v, loaded)``.
A round whose ``use`` stores to global memory keeps the next round's loads
behind it (they may alias), so the rounds' loads would go out in series. A
round past n loads element n - 1 (in range) and uses nothing."""
rounds = -(-n // THREADS)
for b0 in range_constexpr(0, rounds, batch):
idx = range(b0, min(b0 + batch, rounds))
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
for i, g in zip(idx, got):
v = tid + THREADS * i
if v < n:
use(v, g)
def mfma_bf16(a, b, c):
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
def row_sum(red, r, t, waves=None):
"""Row r, token t of the waves' 16x16 MFMA accumulators (wave w's at ``red``
+ (w * 64 + lane) * 4), summed in wave order from 0 (``waves``: which, in
order; all of them by default)."""
tot = fx.Float32(0.0)
for w in waves if waves is not None else range(WAVES):
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
return tot
def ballot(pred):
"""The wave's lanes where ``pred`` holds, a bit a lane."""
return fx.Int64(rocdl.ballot(T.i64, as_ir_value(pred)))
def popcount(x):
return fx.Int32(fx.ctpop(x))
def lanes_below(bal, lane):
"""How many of the lanes in ballot ``bal`` are below ``lane``."""
return popcount(bal & ((fx.Int64(1) << fx.Int64(lane)) - 1))
def load_ptr64(table, i):
"""Entry ``i`` of an i64 table in memory, wave-uniform (each half through
``uniform``): a peer's buffer base, a kernel argument block's pointer."""
pw = fx.Vector(bo.buffer_load(rsrc(table), i * 2, vec_width=2, dtype=T.i32))
return (fx.Int64(uniform(pw[1])) << 32) | fx.Int64(fx.Uint32(uniform(pw[0])))
def bf16_round(a):
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
def bf16_pair(a, b):
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
return (
fx.Vector.from_elements([a, b], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Float32)[0]
)
def bf_lo(w):
"""The f32 of a packed bf16 pair word's low half."""
return (w << 16).bitcast(fx.Float32)
def bf_hi(w):
"""The f32 of a packed bf16 pair word's high half."""
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
def bf2_f32(w):
"""Packed bf16 pair word -> (f32 low, f32 high)."""
return bf_lo(w), bf_hi(w)
def fp8_pack4(a, b, c, d):
"""Four f32 (already in range) -> one dword of four E4M3 bytes, RNE."""
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
# ---------------------------------------------------------------- TP ranks
def peer_bases(peers, tp):
"""Every rank's peer buffer base from the ``peers`` table (an i64 a rank),
loaded once: a put's address must not wait on a load of its own."""
return [load_ptr64(peers, p) for p in range(tp)]
def sum_partials(poll, own, pair_of, tp):
"""The ``tp`` ranks' bf16 pair partials at ``pair_of(src)`` of the mailbox
region ``own``, summed in fp32 in rank order 0 .. tp - 1: one poll batch ->
(lo sum, hi sum). Every rank sums in the same order, so they agree."""
ws = poll([(own, pair_of(src), 1) for src in range(tp)])
acc0, acc1 = bf2_f32(ws[0][0])
for src in range_constexpr(1, tp):
lo, hi = bf2_f32(ws[src][0])
acc0 = acc0 + lo
acc1 = acc1 + hi
return acc0, acc1
# ---------------------------------------------------------------- timeline stamps
def stamp(on, tls, tid, k):
# compile-time gate outside, traced condition inside: they cannot be one `and`
if const_expr(on): # noqa: SIM102
if tid == 0:
fx.ptr_store(memrealtime(), tls + k)
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/runtime/consensus.py
# atom/mono/runtime/peer_memory.py
"""Symmetric peer memory for the in-kernel TP all-reduces, and the agreement every
rank must reach before anything collective (a rank that turned the mono layer
off alone would leave the others spinning on the GPU)."""
import torch
import torch.distributed as dist
from aiter.ops.flydsl.quick_allreduce_int4_ipc import UncachedIpcHeap
# ---------------------------------------------------------------- TP agreement
class MonoUnsupported(Exception):
"""The loaded model or runtime configuration is outside what mono serves."""
def tp_agree(ok: bool, group) -> bool:
"""True when every rank of ``group`` passed ``ok`` true. Every rank must call
it at the same point; ``group`` is a CPU (gloo) process group, or None for a
single rank."""
if group is None:
return ok
flag = torch.tensor([int(ok)], dtype=torch.int32)
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group)
return bool(flag.item())
# ---------------------------------------------------------------- peer buffers
class _DeviceBytes:
"""``__cuda_array_interface__`` of ``nbytes`` bytes at a raw device address, for a
non-owning torch view."""
def __init__(self, ptr: int, nbytes: int):
self.__cuda_array_interface__ = {
"shape": (nbytes,),
"typestr": "|u1",
"data": (ptr, False),
"version": 3,
}
class PeerBuffer:
"""One uncached buffer per rank, every rank holding every peer's address
(``addresses``, int64, indexed by TP rank). What lives where, and why no rank
overwrites a peer's unread data, is the model's layout's to say.
The constructor is collective over ``group`` (the handle exchange, then an
agreement that every rank mapped every peer): every rank must reach it, so a
runner reaches it only after ``tp_agree``. A failed mapping on any rank raises
``MonoUnsupported`` on all of them.
"""
def __init__(
self,
nbytes: int,
group,
rank: int,
npes: int,
device: torch.device,
):
self.local = UncachedIpcHeap.alloc_uncached(nbytes)
# a view, not an owner: close() frees the memory under it
self.bytes = torch.as_tensor(_DeviceBytes(self.local, nbytes), device=device)
self.rank = rank
self._opened: list[int] = []
addresses = [self.local]
if npes > 1:
handles = UncachedIpcHeap.gather_object_list_via_broadcast(
group, UncachedIpcHeap.get_mem_handle_bytes(self.local)
)
addresses, failure = [], None
try:
for peer, handle in enumerate(handles):
if peer == rank:
addresses.append(self.local)
else:
base = UncachedIpcHeap.open_mem_handle(handle)
self._opened.append(base)
addresses.append(base)
except RuntimeError as err: # every rank must still reach the agreement
failure = err
if not tp_agree(failure is None, group):
self.close()
raise MonoUnsupported(
"peer memory handshake failed on "
f"{'this' if failure else 'another'}"
" TP rank"
) from failure
self.addresses = torch.tensor(addresses, dtype=torch.int64, device=device)
def close(self) -> None:
"""Close the peer mappings and free the local buffer; idempotent. Only once no
kernel of any rank can still touch it (the runner is being dropped)."""
opened, self._opened = self._opened, []
for base in opened:
UncachedIpcHeap.close_mem_handle(base)
if self.local:
self.bytes = None
UncachedIpcHeap.free_device_mem(self.local)
self.local = 0
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/plan/execution.py
# atom/mono/plan/layout.py
# atom/mono/plan/arith.py
# atom/mono/plan/shard.py
# atom/mono/plan/build_key.py
"""Plain-Python planning of the mono kernels: the execution model (one CTA a CU,
all resident), mailbox layouts, arithmetic that runs on ints and traced values
alike, TP shards, and build keys (every build parameter, and this package's
sources, in FlyDSL's JIT cache key)."""
import enum
import functools
import hashlib
from dataclasses import dataclass, fields
from pathlib import Path
# ---------------------------------------------------------------- execution model
BLOCKS = 256
THREADS = 512
WAVES = THREADS // 64
def first_task(bid, base):
"""CTA ``bid``'s first task of a stage placed from CTA ``base`` (any count of
CTAs past 0: placements that add task counts run past the grid), then every
BLOCKS-th."""
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
# ---------------------------------------------------------------- mailbox layouts
PAIR_BYTES = 8 # a (value, tag) pair
def pair_layout(items, align: int = PAIR_BYTES, start: int = 0) -> dict:
"""``items`` (name, pairs) laid out in order: name -> (byte offset, bytes),
each region from an ``align``-byte boundary at or past ``start``."""
out, off = {}, start
for name, pairs in items:
off = (off + align - 1) // align * align
out[name] = (off, pairs * PAIR_BYTES)
off += pairs * PAIR_BYTES
return out
# ---------------------------------------------------------------- arithmetic
def sel(pred, a, b):
"""A if pred else b: a conditional on Python values, ``select`` on traced
ones (rewrapped in the traced operand's type)."""
if isinstance(pred, bool):
return a if pred else b
out = pred.select(a, b)
traced = [x for x in (a, b) if not isinstance(x, int)]
return type(traced[0])(out) if traced else out
def imin(a, b):
return sel(a < b, a, b)
def i32(x):
"""``x``, asserted inside the device's Int32 range when it is a Python int."""
if isinstance(x, int):
assert -(2**31) <= x < 2**31, x
return x
def cdiv(a, b):
"""ceil(a / b) for a >= 0, b > 0."""
return i32((a + b - 1) // b)
# ---------------------------------------------------------------- TP shards
class ShardError(ValueError):
"""A width the TP size does not divide, or a TP size the model refuses."""
@dataclass(frozen=True)
class Shard:
tp: int
def __post_init__(self):
if self.tp < 1:
raise ShardError(f"TP {self.tp}")
def split(self, full: int, what: str) -> int:
"""``full`` / tp, refusing a remainder: a rank's share of ``what``."""
if full % self.tp:
raise ShardError(f"{what} {full} is not divisible by TP {self.tp}")
return full // self.tp
def padded(n: int, align: int) -> int:
return cdiv(n, align) * align
# ---------------------------------------------------------------- build keys
_SCALARS = (int, float, bool, str, enum.Enum)
_ROOT = Path(__file__).resolve().parents[1] # the mono package
@functools.cache
def source_digest(*paths: str) -> str:
"""sha256 of every ``.py`` file under ``paths`` (files or directories,
relative to the mono package), each file's path and contents: one model's
kernel sources, so an edit to another model's rebuilds none of these."""
h = hashlib.sha256()
for rel in paths:
root = _ROOT / rel
files = [root] if root.is_file() else sorted(root.rglob("*.py"))
if not files:
raise FileNotFoundError(f"no kernel sources at {root}")
for f in files:
h.update(str(f.relative_to(_ROOT)).encode())
h.update(f.read_bytes())
return h.hexdigest()[:16]
def key_tuple(key, sources: str) -> tuple:
"""``key``'s (field name, value) pairs, every value a scalar, then
("sources", ``sources``): the model's ``source_digest``."""
out = []
for f in fields(key):
value = getattr(key, f.name)
if not isinstance(value, _SCALARS):
raise TypeError(
f"{type(key).__name__}.{f.name} = {value!r}: a build parameter must be"
" a scalar to reach the JIT cache key"
)
out.append((f.name, value))
out.append(("sources", sources))
return tuple(out)
@@ -1,177 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/sync.py
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
between GPUs (system scope), every address a named region (``Addr``)."""
from dataclasses import dataclass
from enum import Enum
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import (
CM_DEV,
CM_SYS,
bf16_pair,
rsrc,
traced,
)
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
@traced
def _spin(load_all, pending):
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
loop keeps the loads from being hoisted."""
v = load_all()
while pending(v):
rocdl.s_sleep(1)
v = load_all()
return v
class Space(Enum):
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
@dataclass(frozen=True)
class Addr:
"""A mailbox region's address: the region's name and space, and the traced
address. The space sets the hand-off's scope: device for SCRATCH, system for
PEER."""
region: str
space: Space
value: object
def sreg(base, offset, region):
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
def preg(base, offset, region):
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
``base``."""
return Addr(region, Space.PEER, base + fx.Int64(offset))
# the store / load cache policy of a hand-off in each space
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
class Mailbox:
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
Every 32-bit value is stored next to this launch's tag; a consumer
polls the payload until all tags match, so a hand-off costs one round trip
(no store drain, no separate flag). The mono layer's launches tag their
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
never carries a live tag. Every address is an ``Addr``: its space picks the
scope.
"""
def __init__(self, tag):
self.tag = tag
def put(self, a, i, v):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
bo.buffer_store(
fx.Vector.from_elements([bits, self.tag], fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def put_bf(self, a, i, vs):
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
store."""
words = []
for j in range_constexpr(len(vs) // 2):
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a.value),
i,
cache_modifier=_SCOPE_CM[a.space],
)
def put_words(self, a, i, words):
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
not kept."""
vals = []
for w in words:
vals += [fx.Int32(w), self.tag]
bo.buffer_store(
fx.Vector.from_elements(vals, fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def poll(self, specs, batch=POLL_MAX):
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
all of one space.
All pairs are re-loaded together while any tag is stale, so a batch costs
one round trip after its last producer lands. Returns the value words."""
if const_expr(len(specs) == 0):
return []
if const_expr(len(specs) > batch):
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
spaces = {a.space for a, _, _ in specs}
assert len(spaces) == 1, f"one poll batch spans {spaces}"
cm = _SCOPE_CM[spaces.pop()]
bases = [a.value for a, _, _ in specs]
tag = self.tag
def load_all():
words = []
for b, (_, i, n) in zip(bases, specs):
w = fx.Vector(
bo.buffer_load(
rsrc(b),
fx.Int32(i) * 2,
vec_width=2 * n,
dtype=T.i32,
cache_modifier=cm,
)
)
words += [w[e] for e in range(2 * n)]
return fx.Vector.from_elements(words, fx.Int32)
nw = sum(2 * n for _, _, n in specs)
def pending(v):
bad = v[1] != tag
for e in range_constexpr(3, nw, 2):
bad = bad | (v[e] != tag)
return bad
v = _spin(load_all, pending)
outs, e = [], 0
for _, _, n in specs:
outs.append([v[e + 2 * q] for q in range(n)])
e += 2 * n
return outs
@traced
def publish(put, a, i, v, who):
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
not overtake the data it announces, and a workgroup-scope release fence does
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
mode: the workgroup's waves share one CU, the readers here do not)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if who:
put(a, i, v)
File diff suppressed because it is too large Load Diff
-464
View File
@@ -1,464 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1-Flash mono decode layer: host side.
One ``DSV41MonoLayer`` a TP rank runs every mono layer of a decode step (M <= 48
rows): two persistent launches a layer (``layer``) from the layer's
inputs at the attention seam to its outputs at the next one -- the MoE's
reduced output, the residual after the FFN seam and that seam's mixes -- with
both TP all-reduces inside the kernels (symmetric peer memory). A layer whose
attention vLLM runs (``ffn``) takes one launch from its unreduced attention
output on. Every argument is a device pointer: the launches can be captured in a
HIP graph. The kernels move their mailbox epoch on themselves; every rank runs
the same launches, so the ranks' epochs agree.
"""
from __future__ import annotations
import os
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
import torch
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
from .common.plan import BLOCKS
from .layer import (
EPOCH_WORDS,
MAX_TOKENS,
MonoBuild,
build_mono_ffn,
build_mono_k1,
build_mono_k2,
peer_half_bytes,
scratch_bytes,
)
from .stages.dims import Dims as MoeDims
from .stages.moe_shape import EXPERTS
__all__ = ["BLOCKS", "MAX_TOKENS", "AttnWeights", "DSV41MonoLayer", "MonoLayerWeights"]
_LOG2E = 1.4426950408889634
HC = 4
def _ensure_writable_flydsl_cache() -> None:
"""Aiter points FlyDSL at its bundled, read-only cache; new kernels need a
writable one."""
cur = os.environ.get("FLYDSL_RUNTIME_CACHE_DIR")
if cur and os.access(cur, os.W_OK):
return
path = Path.home() / ".flydsl" / "cache"
path.mkdir(parents=True, exist_ok=True)
os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = str(path)
def _check_tensors(owner, tag: str, want: dict) -> None:
"""Each named tensor of ``owner`` has the (shape, dtype) the kernels read,
contiguous: their buffer loads are unbounded, so a mismatch would read
garbage rather than fault."""
for name, (shape, dtype) in want.items():
t = getattr(owner, name)
assert tuple(t.shape) == shape and t.dtype == dtype and t.is_contiguous(), (
f"{tag} {name}: {tuple(t.shape)} {t.dtype}, want {shape} {dtype}"
)
@dataclass
class AttnWeights:
"""One layer's attention tensors on this rank, in vLLM's loaded layout."""
layer_id: int
wqkv: torch.Tensor # [1792, 5120] e4m3: [wq_a; wkv]
wqkv_scale: torch.Tensor # [56, 160] uint8 E8M0 (32 x 32 blocks)
q_norm: torch.Tensor # [1280] bf16
kv_norm: torch.Tensor # [512] bf16
wq_b: torch.Tensor # [H * 512, 1280] e4m3
wq_b_scale: torch.Tensor # [H * 16, 40] uint8
wo_a: torch.Tensor # [G * 1024, 4096] e4m3
wo_a_scale: torch.Tensor # [G * 32, 128] uint8
wo_b: torch.Tensor # [5120, G * 1024] e4m3
wo_b_scale: torch.Tensor # [160, G * 32] uint8
attn_sink: torch.Tensor # [>= H] f32
cos_sin: torch.Tensor # [positions, 64] f32 (the layer's rope table)
ratio: int # 0 / 1 / 2: the layer's compress ratio
def check(self, d: Dims) -> None:
h, g = d.heads, d.groups
want = {
"wqkv": ((1792, HIDDEN), torch.float8_e4m3fn),
"wqkv_scale": ((56, HIDDEN // 32), torch.uint8),
"wq_b": ((h * HEAD_DIM, 1280), torch.float8_e4m3fn),
"wq_b_scale": ((h * HEAD_DIM // 32, 40), torch.uint8),
"wo_a": ((g * 1024, d.group_k), torch.float8_e4m3fn),
"wo_a_scale": ((g * 32, d.group_k // 32), torch.uint8),
"wo_b": ((HIDDEN, g * 1024), torch.float8_e4m3fn),
"wo_b_scale": ((HIDDEN // 32, g * 32), torch.uint8),
}
_check_tensors(self, f"layer {self.layer_id}", want)
assert self.cos_sin.dtype == torch.float32 and self.cos_sin.shape[-1] == 64
@dataclass
class MonoLayerWeights:
"""One layer's tensors on this rank, in vLLM's loaded layout (``attn``:
None for a layer whose attention vLLM runs)."""
attn: AttnWeights | None
hc_attn_fn: torch.Tensor # [24, 4 * 5120] f32
hc_attn_scale: torch.Tensor # [3] f32
hc_attn_base: torch.Tensor # [24] f32
attn_norm: torch.Tensor # [5120] bf16
hc_ffn_fn: torch.Tensor
hc_ffn_scale: torch.Tensor
hc_ffn_base: torch.Tensor
ffn_norm: torch.Tensor
gate_w: torch.Tensor # [384, 5120] bf16
bias: torch.Tensor # [384] f32 (e_score_correction_bias)
w13: torch.Tensor # [384, 2 inter, 2560] fp4x2, aiter (16, 16) shuffle
w13_s: torch.Tensor # its scales, aiter shuffle_scale order
w2: torch.Tensor # [384, 5120, inter / 2] fp4x2
w2_s: torch.Tensor
sgu: torch.Tensor # shared gate_up [2 inter, 5120] e4m3, row-major
sgu_s: torch.Tensor # [2 inter / 32, 160] E8M0
sw2: torch.Tensor # shared down [5120, inter] e4m3
sw2_s: torch.Tensor
def check(self, tp: int) -> None:
"""The MoE's tensors: 384 experts, TP-sharded intermediates (the routed
one padded as the loader pads it), AITER's A8W4 layout, 32 x 32 E8M0
blocks for the shared expert."""
d, fp4, e4m3, u8 = (
MoeDims(tp),
torch.float4_e2m1fn_x2,
torch.float8_e4m3fn,
torch.uint8,
)
want = {
"gate_w": ((EXPERTS, HIDDEN), torch.bfloat16),
"bias": ((EXPERTS,), torch.float32),
"w13": ((EXPERTS, 2 * d.inter, HIDDEN // 2), fp4),
"w13_s": ((EXPERTS * 2 * d.inter, HIDDEN // 32), u8),
"w2": ((EXPERTS, HIDDEN, d.inter // 2), fp4),
"w2_s": ((EXPERTS * HIDDEN, d.down_scale_cols), u8),
"sgu": ((2 * d.sh_inter, HIDDEN), e4m3),
"sgu_s": ((2 * d.sh_inter // 32, HIDDEN // 32), u8),
"sw2": ((HIDDEN, d.sh_inter), e4m3),
"sw2_s": ((HIDDEN // 32, d.sh_inter // 32), u8),
}
_check_tensors(self, "MoE", want)
class DSV41MonoLayer:
"""The mono decode runner of one TP rank (``group``: its TP group, for the
peer-memory handle exchange; a gloo / CPU group)."""
def __init__(self, tp: int, rank: int, group, device: torch.device | str = "cuda"):
_ensure_writable_flydsl_cache()
from .common.peer_memory import PeerBuffer
self.tp, self.rank = tp, rank
self.d = Dims(tp)
dev = self.device = torch.device(device)
self._scratch: dict[int, torch.Tensor] = {}
# [epoch, -, -, -, a mark per CTA, the MoE's counters]
self.epoch = torch.zeros(EPOCH_WORDS, dtype=torch.int32, device=dev)
self.q = torch.zeros(
MAX_TOKENS, self.d.heads, HEAD_DIM, dtype=torch.bfloat16, device=dev
)
self.kt = torch.zeros(MAX_TOKENS * KEYS, dtype=torch.int32, device=dev)
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self._first_pre[:, 0] = 1.0
# an index source's K1 outputs, vLLM's indexer's inputs
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
# the attention seam's outputs, K1 -> K2
self.res_mid = torch.zeros(
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
)
self.post_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self.comb_a = torch.zeros(MAX_TOKENS, HC, HC, dtype=torch.float32, device=dev)
self.pre_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self.peer = PeerBuffer(2 * peer_half_bytes(tp), group, rank, tp, dev)
self.peer.bytes.zero_()
self._qk_scale = (
torch.tensor([HEAD_DIM**-0.5 * _LOG2E], dtype=torch.float32)
.view(torch.int32)
.item()
)
self._kernels: dict = {}
def scratch(self, tokens: int) -> torch.Tensor:
"""Step width ``tokens``'s scratch: a width's layout has memory of its own
(``layer.scratch_layout``). Allocated at the width's first step, which is
eager: vLLM runs every graph's batch eagerly before capturing it."""
buf = self._scratch.get(tokens)
if buf is None:
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
f"DSv4.1 mono decode: step width {tokens} first reached inside "
"a CUDA graph capture"
)
buf = torch.zeros(
scratch_bytes(tokens, self.tp), dtype=torch.uint8, device=self.device
)
self._scratch[tokens] = buf
return buf
def first_pre_mix(self, tokens: int) -> torch.Tensor:
return self._first_pre[:tokens]
@staticmethod
def supports(tokens: int) -> bool:
return 1 <= tokens <= MAX_TOKENS
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
key = (tokens, ratio, fold, index)
if key not in self._kernels:
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
return self._kernels[key]
def ffn_kernel(self, tokens: int):
key = (tokens, "ffn")
if key not in self._kernels:
self._kernels[key] = build_mono_ffn(MonoBuild(tokens, self.tp, 0))
return self._kernels[key]
@staticmethod
def _outs(M: int, residual: torch.Tensor) -> tuple:
dev = residual.device
return (
torch.empty(M, HIDDEN, dtype=torch.bfloat16, device=dev),
torch.empty_like(residual),
torch.empty(M, HC, 1, dtype=torch.float32, device=dev),
torch.empty(M, HC, HC, dtype=torch.float32, device=dev),
torch.empty(M, HC, dtype=torch.float32, device=dev),
)
def forward(
self,
w: MonoLayerWeights,
x: torch.Tensor, # [M, 5120] bf16: the previous FFN output (reduced)
residual: torch.Tensor, # [M, 4, 5120] bf16
post_mix: torch.Tensor, # [M, 4, 1] f32
res_mix: torch.Tensor, # [M, 4, 4] f32
pre_mix: torch.Tensor, # [M, 4] f32
positions: torch.Tensor, # [M] int64
slot_mapping: torch.Tensor, # [M] int64
swa_cache: torch.Tensor, # [blocks, block, 584] uint8
swa_indices: torch.Tensor, # [>= M, 1, 128] int32
swa_lens: torch.Tensor, # [>= M] int32
token_to_req: torch.Tensor, # [>= M] int32
topk_indices: torch.Tensor | None = None,
comp_cache: torch.Tensor | None = None,
comp_block_table: torch.Tensor | None = None,
outs: tuple | None = None,
fold: bool = True,
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
):
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
source's top-k selection into ``topk_indices``, run between the
kernels on K1's normed input and qr."""
M = x.shape[0]
assert self.supports(M), M
a = w.attn
assert a is not None
ratio = a.ratio
index = select is not None
assert not index or ratio
k1, k2 = self.kernels(M, ratio, fold, index)
st = torch.cuda.current_stream()
if outs is None:
outs = self._outs(M, residual)
out, res_out, post_out, comb_out, pre_out = outs
for t_ in (x, residual, post_mix, res_mix, pre_mix, positions, slot_mapping):
assert t_.is_contiguous()
assert positions.dtype == slot_mapping.dtype == torch.int64
assert swa_indices.dtype == swa_lens.dtype == token_to_req.dtype == torch.int32
assert swa_indices.stride(0) == 128 and swa_cache.dtype == torch.uint8
assert swa_cache.shape[-1] == 584 and swa_cache.stride(1) == 584
swa_block = swa_cache.shape[1]
if ratio:
assert topk_indices is not None and comp_cache is not None
assert comp_block_table is not None
assert topk_indices.dtype == torch.int32 and topk_indices.stride(0) == 512
assert comp_cache.dtype == torch.uint8 and comp_cache.stride(1) == 584
assert comp_block_table.dtype == torch.int32
comp, comp_stride, comp_block = (
comp_cache,
comp_cache.stride(0),
comp_cache.shape[1],
)
bt, bt_stride, topk = (
comp_block_table,
comp_block_table.stride(0),
topk_indices,
)
else:
comp, comp_stride, comp_block = self._dummy, 0, 1
bt, bt_stride, topk = self._dummy, 0, self._dummy
k1(
residual.data_ptr(),
x.data_ptr(),
post_mix.data_ptr(),
res_mix.data_ptr(),
pre_mix.data_ptr(),
w.hc_attn_fn.data_ptr(),
w.hc_attn_scale.data_ptr(),
w.hc_attn_base.data_ptr(),
w.attn_norm.data_ptr(),
self.res_mid.data_ptr(),
self.post_a.data_ptr(),
self.comb_a.data_ptr(),
self.pre_a.data_ptr(),
a.wqkv.data_ptr(),
a.wqkv_scale.data_ptr(),
a.q_norm.data_ptr(),
a.kv_norm.data_ptr(),
a.wq_b.data_ptr(),
a.wq_b_scale.data_ptr(),
a.cos_sin.data_ptr(),
positions.data_ptr(),
slot_mapping.data_ptr(),
swa_cache.data_ptr(),
swa_cache.stride(0),
swa_block,
self.q.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
comp_block,
bt.data_ptr(),
bt_stride,
topk.data_ptr(),
token_to_req.data_ptr(),
self.kt.data_ptr(),
self.klen.data_ptr(),
self.normed.data_ptr(),
self.qr.data_ptr(),
self.scratch(M).data_ptr(),
self.epoch.data_ptr(),
0,
stream=st,
)
if select is not None:
select(self.normed[:M], self.qr[:M])
k2(
self.q.data_ptr(),
swa_cache.data_ptr(),
swa_cache.stride(0),
swa_block,
comp.data_ptr(),
comp_stride,
comp_block,
self.kt.data_ptr(),
self.klen.data_ptr(),
slot_mapping.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
token_to_req.data_ptr(),
topk.data_ptr(),
bt.data_ptr(),
bt_stride,
positions.data_ptr(),
a.attn_sink.data_ptr(),
self._qk_scale,
a.cos_sin.data_ptr(),
a.wo_a.data_ptr(),
a.wo_a_scale.data_ptr(),
a.wo_b.data_ptr(),
a.wo_b_scale.data_ptr(),
self._zero_rec.data_ptr(),
self.res_mid.data_ptr(),
self.post_a.data_ptr(),
self.comb_a.data_ptr(),
self.pre_a.data_ptr(),
w.hc_ffn_fn.data_ptr(),
w.hc_ffn_scale.data_ptr(),
w.hc_ffn_base.data_ptr(),
w.ffn_norm.data_ptr(),
res_out.data_ptr(),
post_out.data_ptr(),
comb_out.data_ptr(),
pre_out.data_ptr(),
w.gate_w.data_ptr(),
w.bias.data_ptr(),
w.w13.data_ptr(),
w.w13_s.data_ptr(),
w.w2.data_ptr(),
w.w2_s.data_ptr(),
w.sgu.data_ptr(),
w.sgu_s.data_ptr(),
w.sw2.data_ptr(),
w.sw2_s.data_ptr(),
out.data_ptr(),
self.scratch(M).data_ptr(),
self.peer.local,
self.peer.addresses.data_ptr(),
self.rank,
self.epoch.data_ptr(),
0,
stream=st,
)
return outs
def ffn(
self,
w: MonoLayerWeights,
part: torch.Tensor, # [M, 5120] bf16: this rank's unreduced wo_b output
residual: torch.Tensor, # [M, 4, 5120] bf16, after the attention seam
post_mix: torch.Tensor, # [M, 4, 1] f32, the attention seam's
res_mix: torch.Tensor, # [M, 4, 4] f32
pre_mix: torch.Tensor, # [M, 4] f32
outs: tuple | None = None,
):
"""-> (out, residual, post_mix, res_mix, pre_mix) of a layer whose
attention vLLM ran: the attention's TP reduce, the FFN seam, the MoE and
its all-reduce in one launch."""
M = part.shape[0]
assert self.supports(M), M
for t_ in (part, residual, post_mix, res_mix, pre_mix):
assert t_.is_contiguous()
assert part.dtype == residual.dtype == torch.bfloat16
if outs is None:
outs = self._outs(M, residual)
out, res_out, post_out, comb_out, pre_out = outs
self.ffn_kernel(M)(
part.data_ptr(),
residual.data_ptr(),
post_mix.data_ptr(),
res_mix.data_ptr(),
pre_mix.data_ptr(),
w.hc_ffn_fn.data_ptr(),
w.hc_ffn_scale.data_ptr(),
w.hc_ffn_base.data_ptr(),
w.ffn_norm.data_ptr(),
res_out.data_ptr(),
post_out.data_ptr(),
comb_out.data_ptr(),
pre_out.data_ptr(),
w.gate_w.data_ptr(),
w.bias.data_ptr(),
w.w13.data_ptr(),
w.w13_s.data_ptr(),
w.w2.data_ptr(),
w.w2_s.data_ptr(),
w.sgu.data_ptr(),
w.sgu_s.data_ptr(),
w.sw2.data_ptr(),
w.sw2_s.data_ptr(),
out.data_ptr(),
self.scratch(M).data_ptr(),
self.peer.local,
self.peer.addresses.data_ptr(),
self.rank,
self.epoch.data_ptr(),
stream=torch.cuda.current_stream(),
)
return outs
@@ -1,8 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The mono layer's kernel sources, in every build's JIT cache key
(``common.plan.key_tuple``): an edit anywhere in this package rebuilds."""
from vllm.models.deepseek_v41.amd.mono.common.plan import source_digest
SOURCES = source_digest(".")
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The V4.1 stages the mono layer adapts from ROCm/ATOM: the mHC seams
(``seam``), the MoE (``moe``) and the GEMV / LDS helpers they share (``gemv``)."""
@@ -1,60 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/dims.py
"""A TP rank's share of V4.1's MoE widths: every constant the MoE stages derive
from the TP size, in one place (a build key's ``tp`` picks it)."""
from dataclasses import dataclass
from vllm.models.deepseek_v41.amd.mono.common.plan import Shard, padded
# the model's full widths
MOE_INTER = 2304 # a routed expert's intermediate width
SHARED_INTER = 2304 # the shared expert's
# the loader pads a routed expert's rank share to a multiple of this (``FusedMoE``
# ``pad_align``) with zero weights
MOE_PAD = 128
ROWS = 16 # GEMV rows a task
UG_PART = 32 # intermediate columns an ug task part: one FP4 group
@dataclass(frozen=True)
class Dims:
tp: int
def __post_init__(self):
shard = Shard(self.tp)
shard.split(MOE_INTER, "routed intermediate")
shard.split(SHARED_INTER, "shared intermediate")
@property
def inter_real(self) -> int:
"""A routed expert's real intermediate width on this rank."""
return MOE_INTER // self.tp
@property
def inter(self) -> int:
"""... padded as the loader pads it (the rest zero weights)."""
return padded(self.inter_real, MOE_PAD)
@property
def sh_inter(self) -> int:
"""The shared expert's intermediate width on this rank."""
return SHARED_INTER // self.tp
@property
def shared_tasks(self) -> int:
return self.sh_inter // ROWS
@property
def mid_words(self) -> int:
"""MXFP8 words of a pick's routed intermediate."""
return self.inter // 4
@property
def down_scale_cols(self) -> int:
"""w2_s's e8m0 columns: inter / 32, padded to a multiple of 8
(aiter ``shuffle_scale``)."""
return padded(self.inter // 32, 8)
@@ -1,294 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (its GEMV and LDS helpers).
"""The FP8 GEMV tiles of the seam and MoE stages and the helpers they share:
token tiles, LDS rows of MXFP8 words, scalar loads and the mailbox-to-LDS
copy."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, range_constexpr
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.mx import (
UNIT_SCALE,
mfma_scaled,
)
from vllm.models.deepseek_v41.amd.mono.common.ops import (
CM_NT,
fresh,
rsrc,
traced,
)
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
from vllm.models.deepseek_v41.amd.mono.common.sync import POLL_MAX
ROWS = 16 # GEMV rows a task
TILE = 16 # an MFMA's N: the tokens a GEMV pass takes (``token_tiles``)
def ld_f32(ptr, i):
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
def ld_bf(ptr, i):
return fx.Float32(
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
)
def token_tiles(s):
"""A step's tokens in GEMV passes: [(first token, tokens)]. A GEMV loads its
weights once and runs every pass on them, an x tile in LDS at a time."""
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
def tile_rows(s):
"""The LDS rows an x tile holds."""
return min(s, TILE)
# a token's LDS row of MXFP8 words / E8M0 codes, 16 of which a MFMA B operand
# gathers, is padded by 16 B: at a multiple of 64 words they share a bank group
LDS_PAD = 4
def lds_row(words):
return words + LDS_PAD
def lds_at(u, row):
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
return u + u // row * LDS_PAD
def lds_tail(k):
"""Words a K = ``k`` row's last K step reads past its ``lds_row`` (selected
out): the last row's buffer must hold them."""
return max(0, 32 * -(-k // 128) - lds_row(k // 4))
def plus(t0, v):
"""Token ``v`` of the tile from ``t0`` (a first tile adds nothing; ``t0``
traced: a tile a CTA picks at run time)."""
return v if isinstance(t0, int) and t0 == 0 else v + t0
# ---------------------------------------------------------------- GEMV
def _fp8_step_load(c, r_w, r_ws, k, rg, st, row_major=False):
"""K step st's global operands of rows 16 rg ..: the two 64-column weight
chunks (the upper clamped into the row) and the raw scale word. The (16, 16)
shuffle lays a row group out in 32-column blocks of 16 rows, a chunk two of
them (lanes 0-31, 32-63); ``k`` % 64 = 32 (a last quarter step): the lanes
of a block past the row read its last block. ``row_major`` (vLLM's dense
layout, k % 64 == 0): the same registers from a row-major [N, K] weight --
lane l row 16 rg + l % 16, K bytes 128 st + 16 (l // 16) .. and + 64."""
lane = c["lane"]
j = lane // 16
kg = k // 32
if const_expr(row_major):
assert k % 64 == 0
row = rg * 16 + lane % 16
lo_at = row * (k // 4) + st * 32 + j * 4
hi_at = row * (k // 4) + fx.min(2 * st + 1, k // 64 - 1) * 16 + j * 4
lo = fx.Vector(
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
hi = fx.Vector(
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
if const_expr(k % 64 == 0):
lo_at = (rg * (k // 64) + 2 * st) * 256 + lane * 4
else:
lo_at = rg * (k * 4) + fx.min(4 * st + lane // 32, kg - 1) * 128 + lane % 32 * 4
lo = fx.Vector(
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
if const_expr(k % 64 == 0):
hi_at = (rg * (k // 64) + fx.min(2 * st + 1, k // 64 - 1)) * 256 + lane * 4
else:
hi_at = rg * (k * 4) + fx.min(2 * st + 1, k // 64 - 1) * 256 + lane * 4
hi = fx.Vector(
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
)
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
def _fp8_step_mfma(c, k, xl, xsl, st, ops, acc, live=None, rows=None):
"""``acc`` through K step st's scaled MFMA: ``_fp8_step_load``'s operands
against every token's MXFP8 row in LDS. ``live`` false: a step past the end
(its operands those of the last step), zero at unit scale. ``k`` % 64 = 32:
a lane's lower block past the row is zero on both sides."""
s, lane = c["S"] if rows is None else rows, c["lane"]
lo, hi, sa = ops
kg = k // 32
x_row, s_row = lds_row(k // 4), lds_row(kg)
j = lane // 16
col = fx.min(lane % 16, s - 1)
if const_expr(live is not None):
st = fx.min(st, (k + 127) // 128 - 1)
lo = [live.select(lo[d], fx.Int32(0)) for d in range(4)]
past = (2 * st + 1) * 64 >= k
if const_expr(live is not None):
past = past | ~live
av = fx.Vector.from_elements(
[lo[d] for d in range(4)] + [past.select(fx.Int32(0), hi[d]) for d in range(4)],
fx.Int32,
)
grp = st * 4 + j
xlo = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
xhi = fx.Vector(
fx.ptr_load(
xl + (col * x_row + st * 32 + 16 + j * 4),
result_type=fx.Vector.make_type(4, fx.Int32),
)
)
if const_expr(k % 64 != 0):
over = 4 * st + j // 2 >= kg
av = fx.Vector.from_elements(
[over.select(fx.Int32(0), av[d]) for d in range(4)]
+ [av[4 + d] for d in range(4)],
fx.Int32,
)
xlo = [over.select(fx.Int32(0), xlo[d]) for d in range(4)]
xv = fx.Vector.from_elements(
[xlo[d] for d in range(4)]
+ [past.select(fx.Int32(0), xhi[d]) for d in range(4)],
fx.Int32,
)
sb = fx.ptr_load(xsl + (col * s_row + fx.min(grp, kg - 1)))
unit = grp >= kg
if const_expr(live is not None):
unit = unit | ~live
sa = unit.select(fx.Int32(UNIT_SCALE), sa)
sb = unit.select(fx.Int32(UNIT_SCALE), sb)
return mfma_scaled(av, xv, acc, sa, sb)
def _gemv_steps(c, k, split, tiled=False):
"""This wave's K steps (a step past the end marked, ``split`` waves past
split repeat the last one's; ``tiled``: wave w takes part w % split of tile
w / split): [(step, live)]."""
wave = c["wave"]
steps = (k + 127) // 128
if const_expr(split is None):
per = (steps + WAVES - 1) // WAVES
ragged = steps % WAVES != 0
return [
(wave + WAVES * i, (wave + WAVES * i < steps) if ragged else None)
for i in range(per)
]
assert split <= WAVES and steps % split == 0
per = steps // split
part = wave % split if tiled else fx.min(wave, split - 1)
return [(part * per + i, None) for i in range(per)]
def gemv_fp8_loads(c, w, ws, k, rg, split=None, tiled=False, row_major=False):
"""The weight operands of ``gemv_fp8``'s rows for this wave, every one in
flight at once (a step's latency each would put them in series): loads
only. A step past the end loads the last step. ``tiled`` (split x tiles =
WAVES): row groups rg .., one a group of ``split`` waves."""
steps = (k + 127) // 128
r_w, r_ws = rsrc(w), rsrc(ws)
if const_expr(tiled):
assert WAVES % split == 0
rg = rg + c["wave"] // split
return [
_fp8_step_load(
c,
r_w,
r_ws,
k,
rg,
st if live is None else fx.min(st, steps - 1),
row_major,
)
for st, live in _gemv_steps(c, k, split, tiled)
]
def gemv_fp8_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=None):
"""``gemv_fp8_loads``' operands against the rows in LDS (``rows``: a token
tile's, every token's by default) ->
``red[wave][lane]``; the scaled 16x16x128 MFMA, chunk kc in dwords 0-3 and kc +
1 in 4-7. ``k`` a multiple of 32: a last half step has its upper chunk zeroed
on both sides at unit scale (a NaN byte past the row must not reach the
MFMA), its loads clamped into the row: the last row group's would read past
the tensor; a last quarter step its lower chunk's upper block too. A step
past the end adds zero at unit scale.
The K order is the original kernel's, which ``row_sum`` completes:
wave w takes steps w, w + 8, ... (``split`` None: aiter's preshuffled group32
GEMM, P0.5 microbenchmark 1), or waves 0 .. split - 1 one contiguous K range
each, the other waves zero (aiter's split-K bmm, whose last split sums the
partials in split order)."""
lane, wave = c["lane"], c["wave"]
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i, (st, live) in enumerate(_gemv_steps(c, k, split, tiled)):
acc = _fp8_step_mfma(c, k, xl, xsl, st, ops[i], acc, live, rows)
if const_expr(split is not None and not tiled):
acc = fx.Vector.from_elements(
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
fx.Float32,
)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
# ---------------------------------------------------------------- mailbox -> LDS
@traced
def poll_copy(c, idx, width, spans, live=None):
"""Mailbox spans [(region, pair_of, n, dst, row)] into LDS: element u (pair
``pair_of(u)``) to dst + u, or ``lds_at(u, row)`` for a span of rows
``row`` words wide. Thread ``idx`` of ``width`` takes u = idx, idx +
width, ..., POLL_MAX a round trip, each batch stored before the next is
polled (holding them all spills past S = 16); a thread past a span's end
re-reads its last element and stores nothing.
``live``: a span's elements the reader needs (traced, <= n), a span each;
a batch past them is not polled and an element past them neither polled
nor written (one past a step's rows is never written)."""
idx = fresh(idx) # its elements' indices not hoisted out of a task loop
if const_expr(live is None):
_poll_store_batches(
c,
[
(region, pair_of, n, (dst, row), idx + width * i)
for region, pair_of, n, dst, row in spans
for i in range((n + width - 1) // width)
],
)
return
for (region, pair_of, n, dst, row), need in zip(spans, live):
chunks = [
(region, pair_of, need, (dst, row), idx + width * i)
for i in range((n + width - 1) // width)
]
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
if chunks[b0][4] < need:
_poll_store_batches(c, chunks[b0 : b0 + POLL_MAX])
@traced
def _poll_store_batches(c, chunks):
"""``poll_copy``'s chunks [(region, pair_of, n, (dst, row), u)], POLL_MAX a
round trip, each batch stored before the next is polled."""
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
batch = chunks[b0 : b0 + POLL_MAX]
got = c["poll"]([(r, p(fx.min(u, n - 1)), 1) for r, p, n, _, u in batch])
for (_, _, n, (dst, row), u), v in zip(batch, got):
at = u if row is None else lds_at(u, row)
if u < n:
fx.ptr_store(v[0], dst + at)
File diff suppressed because it is too large Load Diff
@@ -1,54 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/moe_plan.py
"""Every per-step decision of which MoE ug flags (UGF) and intermediate rows
(MID) the ug stage writes and the down stage reads, defined once.
The route table (``stage_route``) gives each expert with picks a slot of U, its
picks a run from ``slot_first``, and, past one token tile, its picks tiles of
``TILE`` from ``tile_first`` (each tile's slot ``tile_slot``). An ug task covers
a unit -- a slot, or past one token tile a pick tile -- and one of ``ug_parts`` parts
of the intermediate; its flag is ``ug_flag``. A down round polls each of its
slots' flags (``polled_tile`` its tiles) before it reads their MID rows. Called
by the kernel on traced Int32 and by the CPU property test on Python ints.
"""
from vllm.models.deepseek_v41.amd.mono.common.plan import cdiv, imin
TILE = 16 # picks a pick tile: an MFMA's N
def pick_tiles(count):
"""A slot's pick tiles: its ``count`` picks, ``TILE`` a tile."""
return cdiv(count, TILE)
def ug_parts(groups, ug_groups):
"""A unit's ug parts: the expert's ``groups`` FP4 groups of its real width,
``ug_groups`` a part. A part wholly in the padding is not run: MID's padded
columns stay the step's zeros, against zero weights in down."""
return cdiv(groups, ug_groups)
def ug_flag(unit, part, ug_parts):
"""The UGF pair of ug task (``unit``, ``part``), ``ug_parts`` parts a unit."""
return unit * ug_parts + part
def ug_tasks(units, ug_parts):
"""A step's ug tasks: every part of each of its ``units``."""
return units * ug_parts
def tile_pick0(slot_first, tile, tile_first):
"""Pick tile ``tile``'s first pick, its slot's picks from ``slot_first`` and
tiles from ``tile_first``."""
return slot_first + (tile - tile_first) * TILE
def polled_tile(tile_first, tiles, j):
"""The pick tile a down round polls as a slot's ``j``-th, the slot's
``tiles`` tiles from ``tile_first``: the last past them."""
return tile_first + imin(j, tiles - 1)
@@ -1,229 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/moe_shape.py
"""The MoE stages' build key and route shape: what they are built for (``MoeBuild``),
and the task counts, placements and LDS route table that follow from its
routing (``RouteShape``), with the table's readers. Plain Python but for the
readers; the kernel's stages are in ``moe``."""
from dataclasses import dataclass, field
import flydsl.expr as fx
from vllm.models.deepseek_v41.amd.mono.common.plan import BLOCKS, WAVES
from vllm.models.deepseek_v41.amd.mono.stages import moe_plan as mplan
from vllm.models.deepseek_v41.amd.mono.stages.dims import UG_PART, Dims
from vllm.models.deepseek_v41.amd.mono.stages.gemv import ROWS, TILE, token_tiles
from vllm.models.deepseek_v41.amd.mono.stages.router import PARTS as ROUTER_PARTS
EXPERTS, TOPK = 384, 6 # the target's routing (a build's: ``MoeBuild``)
ROUTER0 = 8 # the router's first task: past the xq tasks (a token each)
MASK_WORDS = 2 # an expert's token bitmap in the route table: tokens <= 64
# a route lane's experts (EXPERTS / 64) sorted descending, ties kept in index
# order: the network by their count
SORT_NETS = {
6: [
(0, 1),
(2, 3),
(4, 5),
(0, 2),
(1, 4),
(3, 5),
(0, 1),
(2, 3),
(4, 5),
(1, 2),
(3, 4),
(2, 3),
],
2: [(0, 1)],
}
@dataclass(frozen=True)
class MoeBuild:
tokens: int
# routed experts and a token's top-k: the target's, or the DSpark draft's
experts: int = EXPERTS
topk: int = TOPK
# the TP size: every width a rank holds (``Dims``)
tp: int = 4
timeline: bool = field(default=False, metadata={"sym": "tl"})
# FP4 groups an ug task (``RouteShape.ug_groups``)
ug_groups: int = 2
@dataclass(frozen=True)
class RouteShape:
"""A build's routing: ``experts`` routed experts, ``topk`` a token of
``tokens``, the task counts and placements that follow, and the route table
``load_route`` lays out in LDS: every token's (ids, weights), |U|, the
experts of U, each (t, k)'s slot in U, each expert's slot, the picks in slot
order and each slot's first position in it (past one token tile also each
slot's first pick tile and each pick tile's slot)."""
experts: int
topk: int
tokens: int
tp: int
# FP4 groups (32 intermediate columns) an ug task, its waves splitting K in
# halves (2) or quarters (1). 1 is twice the tasks, so the CTAs' ug ends
# closer together: faster at the draft's widths (40 rows -4 us), slower at
# the target's (48 rows +3 us; the runners pick)
ug_groups: int = 2
@property
def dims(self) -> Dims:
return Dims(self.tp)
@property
def ug_parts(self) -> int:
"""An expert's ug tasks (``moe_plan.ug_parts``)."""
return mplan.ug_parts(self.dims.inter_real // UG_PART, self.ug_groups)
@property
def ug_k_slices(self) -> int:
"""The K slices of an ug task's waves: a wave a (group, 16-column half,
slice)."""
return WAVES // (2 * self.ug_groups)
@property
def picks(self) -> int:
return self.tokens * self.topk
@property
def max_u(self) -> int:
return min(self.picks, self.experts)
@property
def router_groups(self) -> tuple[int, int]:
"""``stage_router``'s token groups and their tokens: a group a token
tile, as many as one round of CTAs holds (a CTA running two serialized
them: the route waited on the 32 that did at 48 rows)."""
per_group = self.experts // ROWS * ROUTER_PARTS
groups = max(1, min(len(token_tiles(self.tokens)), BLOCKS // per_group))
tokens = -(-self.tokens // groups)
assert (groups - 1) * tokens < self.tokens
return groups, tokens
@property
def router_tasks(self) -> int:
# (token group, 16-row tile, K part): ``stage_router``
return self.router_groups[0] * self.experts // ROWS * ROUTER_PARTS
@property
def router0(self) -> int:
"""Past the xq tasks (a token each from CTA 0)."""
return max(ROUTER0, self.tokens)
@property
def route0(self) -> int:
return self.router0 + self.router_tasks
@property
def shared0(self) -> int:
return self.route0 + max(8, self.tokens)
@property
def shared_units(self) -> int:
"""The shared expert's (task, token tile) units (``stage_shared``)."""
return self.dims.shared_tasks * len(token_tiles(self.tokens))
@property
def ug0(self) -> int:
"""The first ug round's placement: the shared units' CTAs last."""
return self.shared0 + self.shared_units
@property
def sort_net(self):
return SORT_NETS[self.experts // 64]
@property
def tab_words(self) -> int:
return self.tile_slot0 + self.max_tiles if self.multi else self.queue0 + 1
@property
def multi(self) -> bool:
"""More than one token tile: an ug task is a (tile of a slot's picks,
part) (``stage_ug``)."""
return self.tokens > TILE
@property
def slot_tiles(self) -> int:
"""A slot's pick tiles at most (a token picks an expert once)."""
return len(token_tiles(self.tokens))
@property
def max_tiles(self) -> int:
"""The pick tiles of U at most: a slot's partial tile each, then the
whole tiles, and never more than a tile a pick."""
return min(self.picks, self.max_u + self.picks // TILE)
@property
def tile_first0(self) -> int:
"""Past one token tile: each slot's first pick tile, then their count."""
return self.queue0 + 1
@property
def tile_slot0(self) -> int:
"""Past one token tile: each pick tile's slot of U."""
return self.tile_first0 + self.max_u + 1
@property
def queue0(self) -> int:
"""The ug task this CTA took from the queue (``take_ug_task``)."""
return self.mask0 + MASK_WORDS * self.experts
@property
def expert_slot0(self) -> int:
"""The route table's per-expert slots (a slot where the expert is in U)."""
return 3 * self.picks + 1 + self.max_u
@property
def order0(self) -> int:
"""The picks (t topk + k) in slot order, a slot's ascending."""
return self.expert_slot0 + self.experts
@property
def first0(self) -> int:
"""Each slot's first position in the pick order, then the pick count."""
return self.order0 + self.picks
@property
def mask0(self) -> int:
"""Each expert's tokens that picked it, a bit a token (``MASK_WORDS``)."""
return self.first0 + self.max_u + 1
@property
def tile_picks(self) -> int:
"""A slot's picks at most: a token picks an expert once."""
return min(self.tokens, 16)
def route_w(self, tab, s, t, k):
return fx.ptr_load(tab + (t * 2 * self.topk + self.topk + k)).bitcast(
fx.Float32
)
def n_union(self, tab, s):
return fx.ptr_load(tab + 2 * s * self.topk)
def union_expert(self, tab, s, u):
return fx.ptr_load(tab + (2 * s * self.topk + 1 + u))
def slot_first(self, tab, u):
return fx.ptr_load(tab + (self.first0 + u))
def pick_at(self, tab, i):
return fx.ptr_load(tab + (self.order0 + i))
def tile_first(self, tab, u):
return fx.ptr_load(tab + (self.tile_first0 + u))
def tile_slot(self, tab, i):
return fx.ptr_load(tab + (self.tile_slot0 + i))
def route_shape(key: MoeBuild) -> RouteShape:
return RouteShape(key.experts, key.topk, key.tokens, key.tp, key.ug_groups)
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/model_ops/deepseek_v41/router.py
"""V4.1's router GEMV tile for decode: bf16 x [T, 5120] times the gate weight
[384, 5120] -> f32 logits, in one fixed summation order.
Rows 16 a tile, K in ``PARTS`` contiguous parts, 16x16x32 bf16 MFMA: in part
p wave w takes the part's K steps w, w + 8, ... (32 columns each) in that
order, and the eight wave partials are summed in wave order
(``common.ops.row_sum``); the MoE's route stage sums the parts in part
order. The MoE stages spread a tile's parts over PARTS CTAs, and past 16 tokens
a token tile a CTA too, which the order allows: a CTA alone cannot read a
tile, or every token's x, fast enough.
"""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, range_constexpr
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import mfma_bf16, rsrc
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
HIDDEN = 5120
ROWS = 16 # a tile's rows: the MFMA's M
PARTS = 4 # contiguous K parts of a tile, summed in order
def _k0s(part, lane, wave):
"""The K offsets of this lane's loads in part ``part`` (8 bf16 each)."""
per_wave = HIDDEN // 32 // WAVES // PARTS
assert per_wave * 32 * WAVES * PARTS == HIDDEN
return [
(part * per_wave * WAVES + wave + WAVES * i) * 32 + 8 * (lane // 16)
for i in range(per_wave)
]
def router_weight_loads(w, task, part, lane, wave, w_cm=0):
"""``router_tile``'s weight operands (rows 16 task .., K part ``part``).
``w_cm``: their load cache policy (nontemporal where nothing else of the
step reuses them)."""
row = task * ROWS + lane % 16
return [
fx.Vector(
bo.buffer_load(
rsrc(w),
(row * HIDDEN + k0) // 2,
vec_width=4,
dtype=T.i32,
cache_modifier=w_cm,
)
)
for k0 in _k0s(part, lane, wave)
]
def router_x_loads(x, part, s, lane, wave, x_cm=0, t0=0):
"""``router_tile``'s x operands: the s tokens from ``t0`` (column
lane % 16; ``t0`` traced or an int). ``x_cm``: their load cache policy
(device scope when another CTA of the same launch wrote x)."""
tok = fx.min(lane % 16, s - 1)
if const_expr(not isinstance(t0, int) or t0 != 0):
tok = tok + t0
return [
fx.Vector(
bo.buffer_load(
rsrc(x),
(tok * HIDDEN + k0) // 2,
vec_width=4,
dtype=T.i32,
cache_modifier=x_cm,
)
)
for k0 in _k0s(part, lane, wave)
]
def router_tile_mfma(wvs, xvs, lane, wave, red):
"""``router_tile``'s MFMAs, K steps in order -> ``red[wave][lane]``."""
acc = fx.Vector.filled(4, 0.0, fx.Float32)
for i in range_constexpr(len(wvs)):
acc = mfma_bf16(wvs[i].bitcast(fx.BFloat16), xvs[i].bitcast(fx.BFloat16), acc)
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
@@ -1,412 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (slice, gate, norm) and
# atom/models/deepseek_v41/mono/kernels/attn_post.py (the attention's TP reduce).
"""The mono layer's mHC seam stages, each a task table of a persistent launch
handing off through the tagged mailbox:
slice (160, 32 hidden columns x 4 streams each): the owed post folded into
the residual (R_new, bf16 out), the collapsed layer input (mailbox
LIN) and the partial mix projection / sum of squares (mailbox PMIX)
gate (S): the 160 partials -> post / comb (Sinkhorn) and the next pre
norm (S): the RMSNorm -> the bf16 row, handed to the caller's store
reduce (160): the FFN seam's attention output, the TP ranks' partials
(pushed to every rank's ATTN region) summed in the all-reduce's
order -> ``pend_lds``, where its slice reads it
push (160): a rank's partial given in memory (an attention vLLM ran)
to every rank's ATTN region, ahead of the slice's reduce
Every elementwise step follows aiter's ``mhc_fused_post_pre_delayed_rmsnorm``
rounding; the mix projection and the sums of squares, split-K reductions,
differ only in their split and order.
"""
import math
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr import math as fmath
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import (
batched_rounds,
bf16_round,
bf_hi,
bf_lo,
butterfly,
hw_exp2,
hw_rcp,
hw_rsq,
lane_gather,
rsrc,
sum_partials,
traced,
)
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES, pair_layout
from vllm.models.deepseek_v41.amd.mono.common.sync import preg
from vllm.models.deepseek_v41.amd.mono.stages.gemv import (
ld_bf,
ld_f32,
plus,
token_tiles,
)
HC = 4
HIDDEN = 5120
K_MIX = HC * HIDDEN
MIX = HC * (HC + 2) # 24 projections: pre 4 | post 4 | comb 16
EPS = 1e-20
HC_EPS = 1e-6
SINKHORN = 20
POST_MULT = 2.0
LOG2E = 1.0 / math.log(2.0)
COLS = 32 # hidden columns a slice task
SLICES = HIDDEN // COLS # 160
KT = HC * COLS # projection columns a slice task
PART = MIX + 1 # a slice's partials a token: the mixes, then the sum of squares
GATE_BLOCK = 8 # slices a gate thread sums
def scratch_layout(tokens: int) -> dict[str, tuple[int, int]]:
"""Mailbox region -> (byte offset, bytes) of a seam; 8 B a pair."""
pairs = {
"lin": tokens * HIDDEN // 2, # bf16 pairs
"pmix": SLICES * tokens * PART,
}
return pair_layout(pairs.items())
def attn_peer_bytes(tokens: int, tp: int) -> int:
"""ATTN: ``[source rank][token][hidden / 2]`` bf16 pairs."""
return tp * tokens * HIDDEN // 2 * 8
def attn_region_pair(src, t, s, col):
return (src * s + t) * (HIDDEN // 2) + col // 2
def fma(a, b, c):
return fx.Float32(fmath.fma(a, b, c))
def sigmoid(z):
"""Triton's ``tl.sigmoid``: 1 / (1 + exp2(-z log2(e)))."""
return fx.Float32(1.0) / (fx.Float32(1.0) + hw_exp2(z * fx.Float32(-LOG2E)))
def lane_f32(v, src):
return lane_gather(v.bitcast(fx.Int32), src).bitcast(fx.Float32)
def pend_value(c, t, col, cc):
"""The owed post's sublayer output at (token t, column col): read from
``pend``, or where a caller staged it (``pend_lds``, ``[token][32]`` of this
task's columns: a reduction it did first)."""
if const_expr(c.get("pend_lds") is None):
return ld_bf(c["args"]["pend"], t * HIDDEN + col)
return fx.ptr_load(c["pend_lds"] + (t * COLS + cc))
# ---------------------------------------------------------------- slice
@traced
def slice_input(c, task, idx):
"""Thread ``idx``'s pair of the layer input (token idx / 16, columns
32 task + 2 (idx % 16) ..)."""
s, rl = c["S"], c["rl"]
a = c["args"]
c0 = task * COLS
if idx < s * COLS // 2:
t = idx // (COLS // 2)
cc = 2 * (idx % (COLS // 2))
outs = []
for d in range_constexpr(2):
acc = fx.Float32(0.0)
for h in range_constexpr(HC):
rb = fx.ptr_load(rl + (t * KT + h * COLS + cc + d))
acc = acc + ld_f32(a["pre_in"], t * HC + h) * rb
outs.append(acc)
c["put_bf"](c["lin"], t * HIDDEN + c0 + cc, outs)
def slice_at(c0, e):
"""Element e of a slice task at column c0: (token, stream, hidden column)."""
return e // KT, (e % KT) // COLS, c0 + e % COLS
def slice_loads(c, c0, e):
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
a = c["args"]
t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
return r, cm, ld_f32(a["post_in"], t * HC + o)
def slice_row(c, c0, e, got):
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
a = c["args"]
t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
# a settled residual (no owed post): stream o as it arrives
v = got[0]
else:
r, cm, post = got
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
for h in range_constexpr(1, HC - 1):
v = v + cm[h] * r[h]
# bf16 R_new is what the collapse, the sums of squares and the
# mix projection all read
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
@traced
def stage_slice(c, task):
"""R_new, the layer input and the partial projections of hidden columns
32 task .. +32, all four streams, every token."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
rl, fl = c["rl"], c["fl"]
a = c["args"]
c0 = task * COLS
batched_rounds(
tid,
s * KT,
lambda e: slice_loads(c, c0, e),
lambda e, got: slice_row(c, c0, e, got),
)
for i in range_constexpr((MIX * KT + THREADS - 1) // THREADS):
e = tid + THREADS * i
if e < MIX * KT:
j = e // KT
o = (e % KT) // COLS
# the original's two bf16 MFMAs take fn as hi = bf16(fn) and lo =
# bf16(fn - hi): their sum, exact in fp32, is the weight it applies
f = ld_f32(a["hc_fn"], j * K_MIX + o * HIDDEN + c0 + e % COLS)
hi = bf16_round(f)
fx.ptr_store(hi + bf16_round(f - hi), fl + e)
gpu.barrier()
# the layer input: bf16 R_new of the four streams by the incoming pre-mix,
# products rounded, summed in order (the Triton collapse, no contraction);
# two columns a thread, one bf16 pair a mailbox store
for pas in range_constexpr(0, s * COLS // 2, THREADS):
slice_input(c, task, plus(pas, tid))
# sum of squares: token t on wave t % WAVES, two columns a lane
for t in range_constexpr(s):
if wave == t % WAVES:
x0 = fx.ptr_load(rl + (t * KT + 2 * lane))
x1 = fx.ptr_load(rl + (t * KT + 2 * lane + 1))
sq = butterfly(x0 * x0 + x1 * x1, (32, 16, 8, 4, 2, 1))
if lane == 0:
c["put"](c["pmix"], (task * s + t) * PART + MIX, sq)
# mixes: fp32 MFMA 16x16x4, A = fn rows (two 16-row tiles), B = R_new^T,
# a token tile at a time
for t0, n in token_tiles(s):
slice_mix(c, task, t0, n)
@traced
def slice_mix(c, task, t0, n):
"""The mix projection partials of tokens t0 .. t0 + n (``stage_slice``)."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
rl, fl, red = c["rl"], c["fl"], c["red"]
tok = plus(t0, fx.min(lane % 16, n - 1))
kw = KT // WAVES
cs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(2)]
for kk in range_constexpr(kw // 4):
colk = wave * kw + lane // 16 + 4 * kk
b = fx.ptr_load(rl + (tok * KT + colk))
for h in range_constexpr(2):
row = h * 16 + lane % 16
av = fx.ptr_load(fl + (fx.min(row, MIX - 1) * KT + colk))
av = (row < MIX).select(av, fx.Float32(0.0))
ops = rocdl._split_mfma_operands([av, b, cs[h], 0, 0, 0])
cs[h] = fx.Vector(rocdl.mfma_f32_16x16x4f32(T.vec(4, T.f32), *ops).result)
for h in range_constexpr(2):
fx.ptr_store(cs[h], red + ((wave * 2 + h) * 64 + lane) * 4)
gpu.barrier()
if tid < MIX * n:
tl = tid // MIX
t = plus(t0, tl)
j = tid % MIX
h = j // 16
ln = 16 * ((j % 16) // 4) + tl
tot = fx.Float32(0.0)
for w in range_constexpr(WAVES):
tot = tot + fx.ptr_load(red + (((w * 2 + h) * 64 + ln) * 4 + j % 4))
c["put"](c["pmix"], (task * s + t) * PART + j, tot)
gpu.barrier()
# ---------------------------------------------------------------- gate
def _row(v, lane, q):
"""Comb lanes 8..23 hold ``comb[r][c]`` at lane 8 + 4 r + c: element q of
the row."""
return lane_f32(v, (lane - 8) // 4 * 4 + 8 + q)
def _col(v, lane, q):
return lane_f32(v, 8 + 4 * q + (lane - 8) % 4)
@traced
def stage_gate(c, t):
"""Token t's 160 slices, in slice order, -> post / comb / next pre."""
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
red = c["red"]
a = c["args"]
groups = SLICES // GATE_BLOCK # 20
if tid < groups * PART:
g = tid // PART
j = tid % PART
vals = c["poll"](
[
(c["pmix"], ((g * GATE_BLOCK + k) * s + t) * PART + j, 1)
for k in range(GATE_BLOCK)
]
)
acc = vals[0][0].bitcast(fx.Float32)
for k in range_constexpr(1, GATE_BLOCK):
acc = acc + vals[k][0].bitcast(fx.Float32)
fx.ptr_store(acc, red + (g * PART + j))
gpu.barrier()
if wave == 0:
j = fx.min(lane, MIX - 1)
mix = fx.ptr_load(red + j)
sq = fx.ptr_load(red + MIX)
for g in range_constexpr(1, groups):
mix = mix + fx.ptr_load(red + (g * PART + j))
sq = sq + fx.ptr_load(red + (g * PART + MIX))
sc = [ld_f32(a["hc_scale"], i) for i in range(3)]
base = ld_f32(a["hc_base"], j)
# aiter's mhc_fused_post_pre_delayed_rmsnorm reduce: one rstd =
# rsq(fma(sum, 1/K, eps)), then each gate fma(mix rstd, scale, base)
v = mix * hw_rsq(fma(sq, fx.Float32(1.0 / K_MIX), fx.Float32(EPS)))
if lane < HC:
pre = sigmoid(fma(v, sc[0], base))
bo.buffer_store(pre + fx.Float32(HC_EPS), rsrc(a["pre_out"]), t * HC + lane)
if (lane >= HC) & (lane < 2 * HC):
post = sigmoid(fma(v, sc[1], base))
bo.buffer_store(
post * fx.Float32(POST_MULT), rsrc(a["post_out"]), t * HC + lane - HC
)
cv = fma(v, sc[2], base)
row = [_row(cv, lane, q) for q in range(4)]
# Triton's exp: exp2 of (x - max) log2(e)
m = fx.max(fx.max(row[0], row[1]), fx.max(row[2], row[3]))
cv = hw_exp2((cv - m) * fx.Float32(LOG2E))
# fast_dividef is x rcp(y); its first "+ eps" contracts into the fma
row = [_row(cv, lane, q) for q in range(4)]
cv = fma(cv, hw_rcp((row[0] + row[1]) + (row[2] + row[3])), fx.Float32(HC_EPS))
col = [_col(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS))
for _ in range_constexpr(SINKHORN - 1):
row = [_row(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(
((row[0] + row[1]) + (row[2] + row[3])) + fx.Float32(HC_EPS)
)
col = [_col(cv, lane, q) for q in range(4)]
cv = cv * hw_rcp(
((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS)
)
if (lane >= 2 * HC) & (lane < MIX):
bo.buffer_store(cv, rsrc(a["comb_out"]), t * HC * HC + lane - 2 * HC)
gpu.barrier()
# ---------------------------------------------------------------- norm
@traced
def stage_norm(c, t, store):
"""Token t's RMSNorm (rstd = rsq(fma(sum, 1/H, eps)), bf16((x rstd) w));
256 threads, 3 chunks of 8 at 8 t + 2048 c, each chunk's bf16 values handed
to ``store(t, col, ys, live)`` by every thread (``live``: a thread's chunk
in the row; threads 256.. mirror thread 255)."""
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
a = c["args"]
tt = fx.min(tid, 255)
mine = tid < 256
cols = [tt * 8 + 2048 * ch for ch in range(3)]
xs = []
for ch in range_constexpr(3):
base = t * HIDDEN + fx.min(cols[ch], HIDDEN - 8)
words = c["poll"]([(c["lin"], base // 2 + k, 1) for k in range(4)])
v = []
for k in range_constexpr(4):
v += [bf_lo(words[k][0]), bf_hi(words[k][0])]
xs.append([(cols[ch] < HIDDEN).select(x, fx.Float32(0.0)) for x in v])
acc = fx.Float32(0.0)
for ch in range_constexpr(3):
for x in xs[ch]:
acc = acc + x * x
acc = butterfly(acc, (1, 2, 4, 8, 16, 32))
if (lane == 0) & mine:
fx.ptr_store(acc, red + wave)
gpu.barrier()
tot = butterfly(fx.ptr_load(red + lane % 4), (1, 2))
r = hw_rsq(fma(tot, fx.Float32(1.0 / HIDDEN), fx.Float32(EPS)))
for ch in range_constexpr(3):
col = fx.min(cols[ch], HIDDEN - 8)
ys = [
bf16_round((xs[ch][j] * r) * ld_bf(a["norm_w"], col + j)) for j in range(8)
]
store(t, col, ys, mine & (cols[ch] < HIDDEN))
gpu.barrier()
# ---------------------------------------------------------------- reduce
@traced
def stage_reduce(c, task):
"""The attention output at this slice's 32 columns: the TP ranks' partials
in the all-reduce's order (``sum_partials``), in fp32 -> bf16 ->
``pend_lds``."""
s, tid = c["S"], c["tid"]
for pas in range_constexpr(0, s * COLS // 2, THREADS):
reduce_attn_pair(c, task, plus(pas, tid))
gpu.barrier()
@traced
def reduce_attn_pair(c, task, idx):
"""Thread ``idx``'s column pair of token idx / 16 (``stage_reduce``)."""
s = c["S"]
c0 = task * COLS
if idx < s * COLS // 2:
t = idx // (COLS // 2)
col = c0 + 2 * (idx % (COLS // 2))
own = preg(c["sym"], 0, "attn")
acc0, acc1 = sum_partials(
c["poll"],
own,
lambda src: attn_region_pair(src, t, s, col),
c["d"].tp,
)
lds = c["pend_lds"] + (t * COLS + col - c0)
fx.ptr_store(bf16_round(acc0), lds)
fx.ptr_store(bf16_round(acc1), lds + 1)
@traced
def stage_push(c, task, part):
"""This rank's attention partial (``part``: bf16 [S, HIDDEN], wo_b's
unreduced output) at this slice's 32 columns, every token, to every rank's
ATTN region: the pairs ``stage_reduce`` polls, as the bits vLLM wrote."""
s, tid, tp = c["S"], c["tid"], c["d"].tp
for pas in range_constexpr(0, s * COLS // 2, THREADS):
idx = plus(pas, tid)
if idx < s * COLS // 2:
t = idx // (COLS // 2)
col = task * COLS + 2 * (idx % (COLS // 2))
w = bo.buffer_load(
rsrc(part), (t * HIDDEN + col) // 2, vec_width=1, dtype=T.i32
)
pair = attn_region_pair(c["rank"], t, s, col)
for p in range_constexpr(tp):
c["put_words"](preg(c["peer_addr"](p), 0, "attn"), pair, [w])
-435
View File
@@ -1,435 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
A decode step's backbone layer -- the attention seam, the attention, its TP
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
FlyDSL launches (``mono``, this directory) on the layer's loaded
weights, the TP reductions inside the kernels. Every backbone layer of the
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
the first layer and the Engram layers hand K1 a settled residual, and an index
source runs vLLM's compressor and indexer between the two launches. A layer
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
and attention and runs the rest as one launch, the FFN launch, from wo_b's
unreduced output. A step takes the kernels when it is decode only, at most
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
a whole layer also needs causal SWA windows. Anything else runs the layer as
before; the paths share every tensor at the layer boundary, so they interleave
freely.
Every TP rank must take the same path at every layer (the kernels wait on each
other's pushes): the conditions read only metadata every rank holds alike.
"""
from typing import TYPE_CHECKING, Any, cast
import torch
import vllm.envs as envs
from vllm.config import CUDAGraphMode
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterSparseSWAMetadata
from vllm.models.deepseek_v41.sparse_mla import DeepseekV4FlashMLAMetadata
logger = init_logger(__name__)
RECORD = 584 # an fp8_ds_mla KV record: 576 data bytes, 8 scale bytes
SWA_WIDTH = 128 # a causal decode token's window slots
MAX_ROWS = 48 # the kernels' step rows at most: 8 requests x (1 + 5 DSpark drafts)
# the kernels' ISA: CDNA4 (gfx950) scaled MFMA and MX formats
CDNA_VERSIONS = (4,)
_runner = None
def _mono_runner(device: torch.device):
"""The process's mono runner (one a TP rank, shared by every layer). Its
peer-memory handshake is collective over the TP group: every rank builds it
at the same layer of the same step."""
global _runner
if _runner is None:
if torch.cuda.is_current_stream_capturing():
# its peer memory is allocated and exchanged eagerly; vLLM warms
# every graph up eagerly before capturing it
raise RuntimeError(
"--monokernel was first reached inside a CUDA graph "
"capture, before an eager decode step."
)
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
assert MAX_TOKENS >= MAX_ROWS, MAX_TOKENS
_runner = DSV41MonoLayer(
get_tensor_model_parallel_world_size(),
get_tensor_model_parallel_rank(),
get_tp_group().cpu_group,
device,
)
return _runner
def _record_view(kv: torch.Tensor, block: int) -> torch.Tensor:
"""A layer's fp8_ds_mla cache as [blocks, block, RECORD] bytes (a block's
data rows, then its scale words: the kernels index both from it)."""
kv = kv if kv.dtype == torch.uint8 else kv.view(torch.uint8)
assert kv.stride(0) >= block * RECORD, (tuple(kv.shape), kv.stride(), block)
return torch.as_strided(kv, (kv.shape[0], block, RECORD), (kv.stride(0), RECORD, 1))
def _scale_bytes(scale: torch.Tensor) -> torch.Tensor:
return scale if scale.dtype == torch.uint8 else scale.view(torch.uint8)
def _compute_units() -> int:
return current_platform.num_compute_units(torch.accelerator.current_device_index())
def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None:
"""Why the kernels cannot run this deployment's MoE, or None. They read every
one of the 384 experts on each rank, TP-sharded, in AITER's A8W4 MXFP4
layout."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
pc = vllm_config.parallel_config
if pc.enable_expert_parallel or pc.enable_eplb or pc.data_parallel_size > 1:
return "needs tensor parallelism only: no expert or data parallelism"
if envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
return "needs AITER's A8W4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 unset)"
experts = getattr(getattr(layer.ffn, "experts", None), "routed_experts", None)
if experts is None:
return None # not a routed MoE: the layer checks decline it
backend = getattr(experts.quant_method, "mxfp4_backend", None)
if backend != Mxfp4MoeBackend.AITER_MXFP4_BF16:
return f"needs the AITER_MXFP4_BF16 MoE backend, not {backend}"
return None
def _select(
layer: "DeepseekV4DecoderLayer",
positions: torch.Tensor,
normed: torch.Tensor,
qr: torch.Tensor,
) -> None:
"""An index source's compressor and top-k selection, from K1's normed input
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
indexer op, serial."""
attn = layer.attn
comp, indexer = attn.compressor, attn.indexer
assert indexer is not None
if comp is not None:
kv_score = torch.mm(
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
)
latent = comp(kv_score, positions)
comp.insert_cache(latent, positions, attn.rotary_emb)
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
weights, _ = indexer.weights_proj(normed)
q, q_scale, weights = indexer.forward_q(
qr, None, weights, positions, attn.indexer_rotary_emb
)
assert weights is not None
indexer.indexer_op(
normed, (q, q_scale) if q_scale is not None else q, None, weights
)
class MonoDecodeLayer:
"""One decoder layer's mono path (``create``: None when the layer is not
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
launch after vLLM's attention (``ffn``). Its weights are taken from the
layer at the first call, after loading."""
def __init__(self, ffn_only: bool = False) -> None:
self.ffn_only = ffn_only
self._weights: MonoLayerWeights | None = None
@staticmethod
def create(
layer: "DeepseekV4DecoderLayer", vllm_config
) -> "MonoDecodeLayer | None":
if not vllm_config.kernel_config.monokernel:
return None
from vllm.platforms.rocm import get_cdna_version
# the deployment: an explicit opt-in that cannot run is an error
why = None
if get_cdna_version() not in CDNA_VERSIONS:
why = f"needs CDNA{'/'.join(map(str, CDNA_VERSIONS))}"
elif get_tensor_model_parallel_world_size() not in (2, 4):
why = "needs tensor parallel size 2 or 4"
else:
why = _unsupported_moe(layer, vllm_config)
if why is None:
try:
from vllm.models.deepseek_v41.amd.mono.runner import BLOCKS
except ImportError as err:
why = f"needs FlyDSL and AITER's FlyDSL helpers ({err})"
else:
# every CTA of a launch stays resident: a partitioned GPU deadlocks
cus = _compute_units()
if cus < BLOCKS:
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
if why is not None:
raise ValueError(f"--monokernel {why}.")
# the layer: the kernels serve the backbone's seams and MoE
config = vllm_config.model_config.hf_config
attn, ffn = layer.attn, layer.ffn
if layer.use_sequence_parallel:
why = "sequence-parallel layer"
elif not layer.fuse_seam_norm:
why = "needs aiter's fused mHC seam"
elif attn.layer_id >= config.num_hidden_layers:
why = "draft layer"
elif (
ffn.shared_experts is None
or ffn.gate.tid2eid is not None
or ffn.n_routed_experts != 384
or ffn.n_activated_experts != 6
or ffn.routed_scaling_factor != 1.5
or ffn.swiglu_limit != 10.0
or ffn.scoring_func != "sqrtsoftplus"
or not ffn.renormalize
):
why = "MoE routing outside the kernels' (384 / 6, sqrtsoftplus, x1.5)"
if why is not None:
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
return None
# the whole layer: the kernels' attention takes the standard layers; an
# index source's compressor and indexer run between its two launches
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
if attn.compressor is not None and attn.indexer is None:
why = "a compressor without an indexer"
elif attn.indexer is not None and not isinstance(
attn.indexer, DeepseekV41RocmMxfp4Indexer
):
why = f"a {type(attn.indexer).__name__}"
elif attn.compress_ratio not in (0, 1, 2):
why = f"compress ratio {attn.compress_ratio}"
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
why = f"a {attn.kv_cache_dtype} KV cache"
if why is not None:
logger.info_once(
"DSv4.1 mono decode: the FFN launch for layers with %s", why
)
return MonoDecodeLayer(ffn_only=True)
logger.info_once("DSv4.1 mono decode: the whole standard backbone layers")
return MonoDecodeLayer()
def weights(self, layer: "DeepseekV4DecoderLayer"):
if self._weights is not None:
return self._weights
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
a, f = layer.attn, layer.ffn
e, sh = f.experts.routed_experts, f.shared_experts
assert sh is not None # ``create`` took only layers with one
self._weights = MonoLayerWeights(
attn=None if self.ffn_only else self._attn_weights(a),
hc_attn_fn=layer.hc_attn_fn,
hc_attn_scale=layer.hc_attn_scale,
hc_attn_base=layer.hc_attn_base,
attn_norm=layer.attn_norm.weight,
hc_ffn_fn=layer.hc_ffn_fn,
hc_ffn_scale=layer.hc_ffn_scale,
hc_ffn_base=layer.hc_ffn_base,
ffn_norm=layer.ffn_norm.weight,
gate_w=f.gate.weight,
bias=f.gate.e_score_correction_bias,
w13=e.w13_weight,
w13_s=e.w13_weight_scale,
w2=e.w2_weight,
w2_s=e.w2_weight_scale,
sgu=sh.gate_up_proj.weight,
sgu_s=_scale_bytes(sh.gate_up_proj.weight_scale),
sw2=sh.down_proj.weight,
sw2_s=_scale_bytes(sh.down_proj.weight_scale),
)
self._weights.check(get_tensor_model_parallel_world_size())
return self._weights
@staticmethod
def _attn_weights(a):
from vllm.models.deepseek_v41.amd.mono.attention.plan import Dims
from vllm.models.deepseek_v41.amd.mono.runner import AttnWeights
attn = AttnWeights(
layer_id=a.layer_id,
wqkv=a.fused_wqa_wkv.weight,
wqkv_scale=_scale_bytes(a.fused_wqa_wkv.weight_scale),
q_norm=a.q_norm.weight,
kv_norm=a.kv_norm.weight,
wq_b=a.wq_b.weight,
wq_b_scale=_scale_bytes(a.wq_b.weight_scale),
wo_a=a.wo_a.weight,
wo_a_scale=_scale_bytes(a.wo_a.weight_scale),
wo_b=a.wo_b.weight,
wo_b_scale=_scale_bytes(a.wo_b.weight_scale),
attn_sink=a.attn_sink,
cos_sin=a.rotary_emb.cos_sin_cache,
ratio=a.compress_ratio,
)
attn.check(Dims(get_tensor_model_parallel_world_size()))
return attn
@staticmethod
def _skip(why: str) -> None:
logger.debug_once("DSv4.1 mono decode skipped a step: %s", why)
return None
@staticmethod
def _decode_step(layer: "DeepseekV4DecoderLayer", rows: int):
"""(why this step takes the original path -- None for a decode-only step
the kernels take -- the layer's SWA metadata, its compressed cache's)."""
if not 1 <= rows <= MAX_ROWS:
return f"{rows} rows", None, None
if not is_forward_context_available():
return "no forward context", None, None
fc = get_forward_context()
md = fc.attn_metadata
attn = layer.attn
swa, comp = None, None
if isinstance(md, dict):
swa = cast(
"DeepseekV41ROCMAiterSparseSWAMetadata | None",
md.get(attn.swa_cache_layer.prefix),
)
if attn.compressed_cache_prefix is not None:
comp = cast(
"DeepseekV4FlashMLAMetadata | None",
md.get(attn.compressed_cache_prefix),
)
why = None
if fc.cudagraph_runtime_mode not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL):
# a PIECEWISE (breakable) capture is replayed for mixed batches too
why = f"{fc.cudagraph_runtime_mode} step"
elif not isinstance(md, dict): # a profile / dummy run, or DBO ubatches
why = "no attention metadata"
elif swa is None:
why = "layer metadata missing"
elif swa.num_prefills != 0 or swa.num_decodes == 0:
why = "not a decode-only step"
elif swa.num_decode_tokens > rows:
why = "more decode tokens than rows"
return why, swa, comp
def __call__(
self,
layer: "DeepseekV4DecoderLayer",
x: torch.Tensor,
positions: torch.Tensor,
residual: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
pre_mix: torch.Tensor | None,
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
mono kernels, or None when this step takes the original path (always for
an ``ffn_only`` layer)."""
if self.ffn_only:
return None
first = residual is None
if first and x.dim() != 2:
return self._skip("first layer of a pipeline stage")
why, swa, comp = self._decode_step(layer, x.shape[0])
attn = layer.attn
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
if why is None:
# the kernels' attention: causal SWA windows, the compressed cache
if comp is None and attn.compress_ratio:
why = "layer metadata missing"
elif swa.decode_swa_width != SWA_WIDTH:
why = f"SWA width {swa.decode_swa_width}"
elif (
swa.decode_swa_indices is None
or swa.decode_swa_lens is None
or swa.token_to_req_indices is None
):
why = "SWA decode metadata missing"
elif not all(t is not None and t.is_contiguous() for t in tensors):
why = "non-contiguous layer inputs"
if why is not None:
return self._skip(why)
assert swa is not None
runner = _mono_runner(x.device)
logger.info_once(
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
)
# K1's seam folds the owed post into the residual, except where the
# residual arrives settled: the embedding, or after the Engram write
fold = True
if first:
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
pre_mix = runner.first_pre_mix(x.shape[0])
post_mix = res_mix = pre_mix # unread without the fold
fold = False
elif layer.engram is not None and engram_hashes is not None:
residual = layer.mhc_post(x, residual, post_mix, res_mix)
hashes = engram_hashes[:, layer.engram.layer_hash_index]
residual = layer.engram(residual, hashes, engram_mask)
fold = False
kw: dict[str, Any] = {}
if attn.indexer is not None:
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
if attn.compress_ratio:
assert comp is not None
kw.update(
topk_indices=attn.topk_indices_buffer,
comp_cache=_record_view(
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
)
return runner.forward(
self.weights(layer),
x,
residual,
post_mix,
res_mix,
pre_mix,
positions,
swa.slot_mapping,
_record_view(attn.swa_cache_layer.kv_cache, swa.block_size),
swa.decode_swa_indices,
swa.decode_swa_lens,
swa.token_to_req_indices,
fold=fold,
**kw,
)
def ffn(
self,
layer: "DeepseekV4DecoderLayer",
part: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor,
res_mix: torch.Tensor,
pre_mix: torch.Tensor,
) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs by the FFN launch from ``part``, wo_b's unreduced
output, and the attention seam's outputs; None when this step takes the
original path (the caller then reduces ``part`` itself)."""
why, _, _ = self._decode_step(layer, part.shape[0])
tensors = (part, residual, post_mix, res_mix, pre_mix)
if why is None and not all(t.is_contiguous() for t in tensors):
why = "non-contiguous layer inputs"
if why is not None:
return self._skip(why)
runner = _mono_runner(part.device)
logger.info_once(
"DSv4.1 mono decode FFN launch active (decode steps of <= %d rows)",
MAX_ROWS,
)
return runner.ffn(self.weights(layer), *tensors)
File diff suppressed because it is too large Load Diff
-356
View File
@@ -1,356 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1 vision variant.
Thin multimodal wrapper around the text-only ``DeepseekV41LLMForCausalLM``:
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
positions; three learned vectors (``image_start`` / ``image_newline`` /
``image_end``) fill the delimiter positions. Every image-span position
carries ``image_token_id`` (129264) in ``input_ids``; the per-position
roles come from the processor's ``types`` tensor (see
``common/mm_preprocess.py``, and ``common/vision.py`` for the tower).
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
the model so the MoE router can apply ``bias_vl`` to image tokens
(``requires_raw_input_tokens``).
"""
from collections.abc import Iterable
from typing import Annotated
import torch
from torch import nn
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
)
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.models.deepseek_v4.common.vision import (
DeepseekV4Aligner,
DeepseekV4ViT,
run_dp_sharded_vision_tower,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from ..common.mm_preprocess import (
IMAGE_PLACEHOLDER,
DeepseekV4VLDummyInputsBuilder,
DeepseekV4VLMultiModalProcessor,
DeepseekV4VLProcessingInfo,
)
from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin
from ..decoder_replay_layers import DecoderReplayLayers
from .model import (
DeepseekV41LLMForCausalLM,
_linear_scale_param_name,
_make_deepseek_v4_weights_mapper,
)
class DeepseekV4VLImagePixelInputs(TensorSchema):
"""ViT patch inputs for one batched set of images.
Dimensions:
- np: Total ViT patches across images (sum of n_vit_h * n_vit_w)
- c: Number of image channels (3)
- p: ViT patch size
- ni: Number of images
- ns: Total image-span positions (sum of n_llm_h * (n_llm_w + 1) + 2)
"""
patches: Annotated[
torch.Tensor, TensorShape("np", 3, "p", "p", dynamic_dims={"np"})
]
# [n_vit_h, n_vit_w] per image
vit_grid: Annotated[torch.Tensor, TensorShape("ni", 2)]
# [n_llm_h, n_llm_w] per image
llm_grid: Annotated[torch.Tensor, TensorShape("ni", 2)]
types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})]
_VL_PREFIX_MAPPING: dict[str, str | None] = {
"layers.": "language_model.model.layers.",
"embed.": "language_model.model.embed.",
"norm.": "language_model.model.norm.",
"hc_head": "language_model.model.hc_head",
"mtp.": "language_model.model.mtp.",
}
def _make_deepseek_v4_vl_weights_mapper(
expert_dtype: str, linear_scale_name: str
) -> WeightsMapper:
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name)
return WeightsMapper(
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
orig_to_new_regex=base.orig_to_new_regex,
orig_to_new_suffix={
"head.weight": "language_model.lm_head.weight",
"embed.weight": "embed_tokens.weight",
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
},
orig_to_new_substr={
".shared_experts.w2": ".shared_experts.down_proj",
# The MTP/DSpark draft heads are not supported for the vision
# variant; drop their weights.
"mtp.": None,
},
)
@MULTIMODAL_REGISTRY.register_processor(
DeepseekV4VLMultiModalProcessor,
info=DeepseekV4VLProcessingInfo,
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
)
class DeepseekV41ForCausalLM(
nn.Module,
DeepseekV4VLEncoderCudaGraphMixin,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsPP,
SupportsEagle3,
):
"""Multimodal entry point for DeepSeek-V4.1 checkpoints with a vision tower.
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
delegates through ``language_model`` via the protocol defaults.
``SupportsEncoderCudaGraph`` (ViT CUDA graph capture/replay) is
implemented by ``DeepseekV4VLEncoderCudaGraphMixin``.
"""
supports_encoder_tp_data = True
# Both of these are read off the *class* by
# ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs,
# to translate a quantization config's checkpoint-style module names into
# runtime prefixes. ``__init__`` later overwrites the instance
# ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the
# prefix rules matter here. Without the class attributes, per-layer specs
# naming ``layers.N.attn.wq_a`` never match this wrapper's
# ``language_model.model.layers.N.attn.wq_a`` and silently fall back to
# the global spec.
packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"},
)
# The MoE router needs raw token ids to detect image-span tokens
# (all carrying image_token_id, see common/mm_preprocess.py) and apply
# bias_vl.
requires_raw_input_tokens = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality == "image":
return IMAGE_PLACEHOLDER
raise ValueError(f"Unsupported modality: {modality!r}")
def __init__(self, *, vllm_config, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config = model_config.hf_config
self.config = config
self.multimodal_config = model_config.multimodal_config
assert self.multimodal_config is not None
# The tower is always built; _mark_tower_model stubs it out
# (StageMissingLayer, weights skipped) when the image limit is 0.
with self._mark_tower_model(vllm_config, {"image"}):
self.use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
self.vision = DeepseekV4ViT(config)
self.aligner = DeepseekV4Aligner(config)
self.image_start = nn.Parameter(
torch.empty(config.hidden_size, dtype=torch.float32)
)
self.image_end = nn.Parameter(
torch.empty(config.hidden_size, dtype=torch.float32)
)
self.image_newline = nn.Parameter(
torch.empty(config.hidden_size, dtype=torch.float32)
)
self.vision.to(dtype=model_config.dtype)
self.aligner.to(dtype=model_config.dtype)
with self._mark_language_model(vllm_config):
self.language_model = DeepseekV41LLMForCausalLM(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "language_model"),
)
# The outer mapper (see load_weights) fully resolves HF names into
# this wrapper's namespace before AutoWeightsLoader strips the
# "language_model." prefix and delegates to the child's load_weights,
# so the child's own mapper must be a no-op. Its suffix rules are not
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
# re-fire "head.weight" -> "lm_head.weight").
self.language_model.hf_to_vllm_mapper = WeightsMapper()
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
self.language_model.make_empty_intermediate_tensors
)
expert_dtype = getattr(config, "expert_dtype", "fp4")
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
expert_dtype, _linear_scale_param_name(vllm_config, expert_dtype)
)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> DeepseekV4VLImagePixelInputs | None:
patches = kwargs.pop("patches", None)
if patches is None:
return None
return DeepseekV4VLImagePixelInputs(
patches=patches,
vit_grid=kwargs.pop("vit_grid"),
llm_grid=kwargs.pop("llm_grid"),
types=kwargs.pop("types"),
resolve_bindings={"p": self.config.vision_patch_size},
)
def _process_image_input(
self,
image_input: DeepseekV4VLImagePixelInputs,
) -> tuple[torch.Tensor, ...]:
patches = image_input.patches.to(self.aligner.w1.weight.dtype)
vit_grid = image_input.vit_grid.tolist()
image_embeds_list: list[torch.Tensor]
if self.use_data_parallel and get_tensor_model_parallel_world_size() > 1:
# Data-parallel ViT: shard images across TP ranks and all-gather
# the per-image embeddings (weights are replicated on every rank).
image_embeds_list = run_dp_sharded_vision_tower(
self.vision, self.aligner, patches, vit_grid
)
else:
image_embeds_list = []
vit_offset = 0
for n_vit_h, n_vit_w in vit_grid:
n_vit = n_vit_h * n_vit_w
image_embeds_list.append(
self._encode_image(
patches[vit_offset : vit_offset + n_vit], n_vit_h, n_vit_w
)
)
vit_offset += n_vit
embeds: list[torch.Tensor] = []
span_offset = 0
for image_embeds, (n_llm_h, n_llm_w) in zip(
image_embeds_list, image_input.llm_grid.tolist(), strict=True
):
span_len = n_llm_h * (n_llm_w + 1) + 2
embeds.append(
self._build_image_span(
image_embeds,
image_input.types[span_offset : span_offset + span_len],
)
)
span_offset += span_len
return tuple(embeds)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
return []
return self._process_image_input(image_input)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
inputs_embeds = self.language_model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
assert is_multimodal is not None
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
@staticmethod
def get_model_state_cls():
from ..nvidia.model_state import DeepseekV41ModelState
return DeepseekV41ModelState
@property
def token_lookback_depth(self) -> int:
return self.language_model.token_lookback_depth
@property
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
return self.language_model.decoder_replay_layers
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors=None,
inputs_embeds: torch.Tensor | None = None,
lookback_token_ids: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
return self.language_model(
input_ids,
positions,
intermediate_tensors,
inputs_embeds,
lookback_token_ids=lookback_token_ids,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits_local(hidden_states)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.language_model.get_expert_mapping()
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
return self.language_model.get_mtp_target_hidden_states()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Map HF names into this wrapper's namespace up front and sort, so
# the "language_model." group reaches the child loader as one
# contiguous block (AutoWeightsLoader delegates per contiguous group,
# and the child's load_weights finalizes fused expert weights, which
# must not run on a partially loaded model).
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
# The child's load_weights already ran its post-load finalization.
self._weights_finalized = True
return loaded_params
def process_weights_after_loading(self) -> None:
# Model-level post-load hook (called by the loader after any load
# format). Under DummyModelLoader the child's load_weights — and
# hence its finalize step — is bypassed, so run it here instead.
if getattr(self, "_weights_finalized", False):
return
self.language_model.process_weights_after_loading()
File diff suppressed because it is too large Load Diff
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More