[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V3.2 (``deepseek_v32``) platform entry point.
|
||||
|
||||
DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
|
||||
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
|
||||
attend. The same model code serves any DSA checkpoint, including GLM-5.2
|
||||
(``glm_moe_dsa``), which reuses this architecture.
|
||||
|
||||
The CUDA and ROCm implementations select capability-specific kernels
|
||||
internally and fall back when an optimization is unavailable. Other platforms
|
||||
use the generic implementation by default.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized
|
||||
# kernels remain gated on the device capabilities they support.
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
elif current_platform.is_rocm():
|
||||
from .amd.model import DeepseekV32ForCausalLM
|
||||
from .amd.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
|
||||
from .amd.mtp import DeepseekV32MTP
|
||||
else:
|
||||
# XPU and CPU keep the generic implementation.
|
||||
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
|
||||
|
||||
__all__ = [
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV32MTP",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
]
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,399 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2ForCausalLM,
|
||||
DeepseekV2MLP,
|
||||
DeepseekV2MoE,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
get_pp_missing_layer_names,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
)
|
||||
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .rocm import DeepseekV32MLAAttention
|
||||
|
||||
|
||||
class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str,
|
||||
config=None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
if config is None:
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
|
||||
layer_idx = int(prefix.split(sep=".")[-1])
|
||||
self.layer_idx = layer_idx
|
||||
self.use_mha = False
|
||||
|
||||
self.self_attn = DeepseekV32MLAAttention(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.zero_mlp_output = False
|
||||
if (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % moe_layer_freq == 0
|
||||
):
|
||||
self.mlp = DeepseekV2MoE(
|
||||
config=config,
|
||||
parallel_config=parallel_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
moe_cfg = self.mlp.experts.moe_config
|
||||
pc = moe_cfg.moe_parallel_config
|
||||
# Validated on MoRI only. DeepEP, pplx and the other combines
|
||||
# keep the previous always-skip behavior.
|
||||
if pc.use_mori_kernels:
|
||||
# Combine is already all-reduced; the next fused norm sums TP.
|
||||
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
|
||||
else:
|
||||
moe_cfg.skip_final_all_reduce = True
|
||||
else:
|
||||
self.mlp = DeepseekV2MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.input_layernorm
|
||||
)
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.zero_mlp_output:
|
||||
hidden_states = torch.zeros_like(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class DeepseekV32Model(torch.nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
self.device = current_platform.device_type
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
self.is_v32 = True
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV32DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV2MoE,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
self.aux_hidden_state_layers = tuple[int, ...]()
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
assert input_ids is not None
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = []
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
hidden_states, _ = fused_allreduce_rms_norm(hidden_states, residual, self.norm)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("fused_qkv_a_proj", "q_a_proj", 0),
|
||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||
("wk_weights_proj", "wk", 0),
|
||||
("wk_weights_proj", "weights_proj", 1),
|
||||
]
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
name = name_mapped
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
|
||||
# When AITER fusion_shared_experts is enabled the checkpoint
|
||||
# provides a single widened shared_experts tensor per layer with
|
||||
# no expert index. Split it evenly and route the slices into the
|
||||
# expert slots appended after the routed experts.
|
||||
num_chunks = 1
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
# gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1).
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
total = loaded_weight.shape[split_dim]
|
||||
assert total % num_chunks == 0, (
|
||||
f"Shared expert weight dim {total} "
|
||||
f"not divisible by num_chunks {num_chunks}"
|
||||
)
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = chunk_name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loader(param, loaded_weight)
|
||||
if name is not None and not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
|
||||
model_cls = DeepseekV32Model
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.moe_layers = []
|
||||
self.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if isinstance(layer.mlp, DeepseekV2MoE):
|
||||
example_moe = layer.mlp
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
@@ -1,425 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2MixtureOfExperts,
|
||||
DeepseekV2MoE,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import DeepseekV32DecoderLayer
|
||||
|
||||
|
||||
class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.mtp_block = DeepseekV32DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
config=config,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
eh_input = fused_eh_norm(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepseekV32MultiTokenPredictorLayer(
|
||||
vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
for layer in self.layers.values():
|
||||
self_attn = getattr(layer.mtp_block, "self_attn", None)
|
||||
if self_attn is not None and hasattr(self_attn, "skip_topk"):
|
||||
self_attn.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, slot_ids: torch.Tensor):
|
||||
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
|
||||
num_slots = slot_ids.numel()
|
||||
for layer in self.layers.values():
|
||||
self_attn = getattr(layer.mtp_block, "self_attn", None)
|
||||
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
|
||||
topk_indices_buffer = self_attn.topk_indices_buffer
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = DeepseekV32MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.set_moe_parameters()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], self.config.hidden_size
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
self.num_expert_groups = self.config.n_group
|
||||
self.moe_layers = []
|
||||
self.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers.values():
|
||||
mlp = layer.mtp_block.mlp
|
||||
if isinstance(mlp, DeepseekV2MoE):
|
||||
example_moe = mlp
|
||||
self.moe_mlp_layers.append(mlp)
|
||||
self.moe_layers.append(mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return (
|
||||
get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
and "embed_tokens" not in name
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("fused_qkv_a_proj", "q_a_proj", 0),
|
||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||
("wk_weights_proj", "wk", 0),
|
||||
("wk_weights_proj", "weights_proj", 1),
|
||||
]
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
# A tied top-level embed_tokens has no spec layer to rewrite
|
||||
# from; the draft needs its own copy under PP.
|
||||
param = params_dict.get(name) if "embed_tokens" in name else None
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
else:
|
||||
name = name_mapped
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
num_chunks = 1
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
total = loaded_weight.shape[split_dim]
|
||||
assert total % num_chunks == 0
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = chunk_name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if (
|
||||
spec_layer != self.model.mtp_start_layer_idx
|
||||
and ".layers" not in name
|
||||
):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers:
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint."
|
||||
)
|
||||
return loaded_params
|
||||
@@ -1,314 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
|
||||
def get_attn_backend(self):
|
||||
return DeepseekV32IndexerBackend
|
||||
|
||||
@property
|
||||
def uses_shuffled_layout(self) -> bool:
|
||||
# aiter's gather/insert pair shuffles the cache above block size 1:
|
||||
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
|
||||
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1
|
||||
|
||||
|
||||
class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
|
||||
indexer_cache_cls = DeepseekV32ROCmIndexerCache
|
||||
|
||||
|
||||
class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
indexer_cls = DeepseekV32ROCmIndexer
|
||||
|
||||
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
|
||||
super().__init__(
|
||||
vllm_config,
|
||||
config,
|
||||
prefix,
|
||||
topk_indices_buffer,
|
||||
attn_backend=ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
self.indexer_op: SparseAttnIndexer | None = None
|
||||
if self.indexer is not None:
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
self.indexer.k_cache,
|
||||
self.indexer.quant_block_size,
|
||||
self.indexer.scale_fmt,
|
||||
self.indexer.topk_tokens,
|
||||
self.indexer.head_dim,
|
||||
self.indexer.max_model_len,
|
||||
self.indexer.max_total_seq_len,
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert=True,
|
||||
)
|
||||
self._fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
self._fp8_kv_needs_view = self._fp8_kv and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_c, k_pe = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
kw = self.indexer.wk_weights_proj(hidden_states)[0]
|
||||
index_k = kw[:, : self.indexer.head_dim]
|
||||
index_weights = kw[:, self.indexer.head_dim :]
|
||||
else:
|
||||
index_k = None
|
||||
index_weights = None
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
output = torch.empty(
|
||||
(num_tokens, self.num_local_heads * self.v_head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
self._fused_attention(
|
||||
positions, q_c, kv_c, k_pe, index_k, index_weights, output
|
||||
)
|
||||
return self.o_proj(output)[0]
|
||||
|
||||
def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
q_nope = q_nope.transpose(0, 1) # (N, tokens, P)
|
||||
|
||||
if self.is_aiter_triton_fp4_bmm_enabled:
|
||||
ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
|
||||
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
|
||||
)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
ql_nope = rocm_aiter_ops.triton_fp8_bmm(
|
||||
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
|
||||
)
|
||||
else:
|
||||
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
|
||||
|
||||
return ql_nope, q_pe
|
||||
|
||||
def _run_indexer(
|
||||
self,
|
||||
q_c: torch.Tensor,
|
||||
index_q_fp8: torch.Tensor | None,
|
||||
index_weights_out: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Run the ROCm sparse indexer (forward_hip) if this layer has an indexer."""
|
||||
if self.indexer_op is not None:
|
||||
self.indexer_op.forward_hip(q_c, index_q_fp8, None, index_weights_out)
|
||||
|
||||
def _build_q_for_attn(
|
||||
self,
|
||||
ql_nope: torch.Tensor,
|
||||
mqa_q: torch.Tensor,
|
||||
num_actual: int,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._fp8_kv:
|
||||
# fp8 KV: mqa_q is the full [ql_nope; q_pe] packed as fp8.
|
||||
return mqa_q[:num_actual]
|
||||
|
||||
return (ql_nope[:num_actual], mqa_q[:num_actual])
|
||||
|
||||
def _compute_uv_out(
|
||||
self,
|
||||
attn_out: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
num_actual: int,
|
||||
) -> None:
|
||||
x = attn_out.view(
|
||||
num_actual, self.num_local_heads, self.kv_lora_rank
|
||||
).transpose(0, 1) # (N, tokens, L)
|
||||
out_view = output[:num_actual].view(
|
||||
num_actual, self.num_local_heads, self.v_head_dim
|
||||
)
|
||||
|
||||
if self.is_aiter_triton_fp4_bmm_enabled:
|
||||
rocm_aiter_ops.batched_gemm_a16wfp4(
|
||||
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
|
||||
)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
rocm_aiter_ops.triton_fp8_bmm(
|
||||
x,
|
||||
self.W_V,
|
||||
self.W_V_scale,
|
||||
group_size=128,
|
||||
transpose_bm=True,
|
||||
YQ=out_view,
|
||||
)
|
||||
else:
|
||||
torch.bmm(x, self.W_UV, out=out_view.transpose(0, 1))
|
||||
|
||||
@eager_break_during_capture
|
||||
def _fused_attention(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_weights: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.layer_name)
|
||||
elif isinstance(attn_metadata_raw, list):
|
||||
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
|
||||
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
has_indexer = True
|
||||
indexer_k_norm_w = self.indexer.k_norm.weight
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
indexer_k_norm_eps = self.indexer.k_norm.eps
|
||||
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
||||
indexer_k_cache = self.indexer.k_cache.kv_cache
|
||||
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
|
||||
indexer_softmax_scale = self.indexer.softmax_scale
|
||||
indexer_n_head_scale = self.indexer.n_head**-0.5
|
||||
else:
|
||||
has_indexer = False
|
||||
indexer_k_norm_w = None
|
||||
indexer_k_norm_bias = None
|
||||
indexer_k_norm_eps = 1e-6
|
||||
indexer_k_rope_cos_sin_cache = None
|
||||
indexer_k_cache = None
|
||||
indexer_cache_shuffled = False
|
||||
indexer_softmax_scale = 0.0
|
||||
indexer_n_head_scale = 0.0
|
||||
|
||||
if attn_metadata is None:
|
||||
mla_kv_cache = None
|
||||
mla_k_scale = None
|
||||
indexer_k_cache = None
|
||||
mla_slot = None
|
||||
else:
|
||||
mla_kv_cache = self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
|
||||
kv_c_out = torch.empty_like(kv_c) if use_mha else None
|
||||
k_pe_out = torch.empty_like(k_pe) if use_mha else None
|
||||
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
self.q_a_layernorm.weight,
|
||||
self.q_a_layernorm.variance_epsilon,
|
||||
kv_c,
|
||||
self.kv_a_layernorm.weight,
|
||||
self.kv_a_layernorm.variance_epsilon,
|
||||
k_pe,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
index_k,
|
||||
indexer_k_norm_w,
|
||||
indexer_k_norm_bias,
|
||||
indexer_k_norm_eps,
|
||||
indexer_k_rope_cos_sin_cache,
|
||||
self.topk_indices_buffer,
|
||||
slot_mapping=mla_slot,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
indexer_cache_shuffled=indexer_cache_shuffled,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=self.kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
kv_c_out=kv_c_out,
|
||||
k_pe_out=k_pe_out,
|
||||
)
|
||||
|
||||
if use_mha:
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split(
|
||||
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
ql_nope = q_nope
|
||||
else:
|
||||
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
|
||||
else:
|
||||
index_q = None
|
||||
|
||||
index_q_fp8, index_weights_out, mqa_q = fused_q(
|
||||
positions,
|
||||
q_pe,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
index_q,
|
||||
self.indexer_rope_emb.cos_sin_cache if has_indexer else None,
|
||||
ql_nope,
|
||||
self._q_scale,
|
||||
index_weights,
|
||||
indexer_softmax_scale,
|
||||
indexer_n_head_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
quantize_mqa=self._fp8_kv and not use_mha,
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
self._run_indexer(q_c, index_q_fp8, index_weights_out)
|
||||
|
||||
if attn_metadata is None:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
if use_mha:
|
||||
assert kv_c_out is not None and k_pe_out is not None
|
||||
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
|
||||
self.forward_impl(
|
||||
mha_q,
|
||||
kv_c_out,
|
||||
k_pe_out.unsqueeze(1),
|
||||
self.kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
return
|
||||
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
kv_cache = self.kv_cache
|
||||
if self._fp8_kv_needs_view:
|
||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual)
|
||||
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
q_for_attn, kv_cache, attn_metadata, self
|
||||
)
|
||||
|
||||
self._compute_uv_out(attn_out, output, num_actual)
|
||||
@@ -1,653 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
SparseAttnIndexer,
|
||||
sparse_attn_indexer,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.ops.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32Indexer(nn.Module):
|
||||
indexer_cache_cls = DeepseekV32IndexerCache
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
hidden_size: int,
|
||||
q_lora_rank: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
cache_config: CacheConfig | None,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.topk_tokens = config.index_topk
|
||||
self.n_head = config.index_n_heads
|
||||
self.head_dim = config.index_head_dim
|
||||
self.rope_dim = config.qk_rope_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
|
||||
self.wq_b = ReplicatedLinear(
|
||||
self.q_lora_rank,
|
||||
self.head_dim * self.n_head,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.wq_b",
|
||||
)
|
||||
|
||||
self.wk_weights_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[self.head_dim, self.n_head],
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
disable_tp=True,
|
||||
prefix=f"{prefix}.wk_weights_proj",
|
||||
)
|
||||
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
|
||||
self.softmax_scale = self.head_dim**-0.5
|
||||
|
||||
self.scale_fmt = "ue8m0"
|
||||
self.quant_block_size = 128
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
assert cache_config is not None, "DeepSeek V3.2 indexer requires cache_config"
|
||||
self.k_cache = type(self).indexer_cache_cls(
|
||||
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
|
||||
dtype=torch.uint8,
|
||||
prefix=f"{prefix}.k_cache",
|
||||
cache_config=cache_config,
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.prefix = prefix
|
||||
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
get_max_prefill_buffer_size,
|
||||
)
|
||||
|
||||
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32Attention(MLAAttention):
|
||||
indexer: "DeepseekV32Indexer | None"
|
||||
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
|
||||
supports_pcp_dcp: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
prefix: str,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
attn_backend: "type | None" = None,
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
hidden_size = config.hidden_size
|
||||
qk_nope_head_dim = config.qk_nope_head_dim
|
||||
qk_rope_head_dim = config.qk_rope_head_dim
|
||||
v_head_dim = config.v_head_dim
|
||||
q_lora_rank = config.q_lora_rank
|
||||
kv_lora_rank = config.kv_lora_rank
|
||||
num_heads = config.num_attention_heads
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert num_heads % tp_size == 0
|
||||
num_local_heads = num_heads // tp_size
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
scaling = qk_head_dim**-0.5
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
|
||||
# DSA checkpoints may use plain ("default") or yarn-scaled RoPE.
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
if config.rope_parameters["rope_type"] == "deepseek_yarn":
|
||||
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = config.rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
scaling = scaling * mscale * mscale
|
||||
|
||||
layer_id = extract_layer_index(prefix)
|
||||
index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
if index_topk_pattern is None:
|
||||
skip_topk = (
|
||||
max(layer_id - index_skip_topk_offset + 1, 0) % index_topk_freq != 0
|
||||
)
|
||||
elif 0 <= layer_id < len(index_topk_pattern):
|
||||
skip_topk = index_topk_pattern[layer_id] == "S"
|
||||
else:
|
||||
skip_topk = False
|
||||
|
||||
num_hidden_layers = getattr(config, "num_hidden_layers", None)
|
||||
is_mtp_layer = num_hidden_layers is not None and layer_id >= num_hidden_layers
|
||||
|
||||
kv_b_proj = ColumnParallelLinear(
|
||||
kv_lora_rank,
|
||||
num_heads * (qk_nope_head_dim + v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
indexer = None
|
||||
if not skip_topk or is_mtp_layer:
|
||||
indexer = type(self).indexer_cls(
|
||||
vllm_config,
|
||||
config,
|
||||
hidden_size,
|
||||
q_lora_rank,
|
||||
quant_config,
|
||||
cache_config,
|
||||
topk_indices_buffer,
|
||||
prefix=f"{prefix}.indexer",
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
num_heads=num_local_heads,
|
||||
scale=scaling,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
q_lora_rank=q_lora_rank,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
kv_b_proj=kv_b_proj,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
use_sparse=True,
|
||||
indexer=indexer,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
attn_backend=attn_backend,
|
||||
)
|
||||
|
||||
self.num_local_heads = num_local_heads
|
||||
self.qk_head_dim = qk_head_dim
|
||||
self.indexer = indexer
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
self.skip_topk = False
|
||||
enable_short_prefill_scoring_skip = (
|
||||
not is_mtp_layer
|
||||
and not skip_topk
|
||||
and not self.use_pcp
|
||||
and current_platform.is_cuda()
|
||||
and self.supports_dense_mha_prefill
|
||||
)
|
||||
self._dense_mha_metadata_layer_name = (
|
||||
self.layer_name if enable_short_prefill_scoring_skip else ""
|
||||
)
|
||||
|
||||
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
|
||||
self._fp8_kv_needs_view = fp8_attention and not self.kv_cache_dtype.endswith(
|
||||
"_ds_mla"
|
||||
)
|
||||
|
||||
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
|
||||
|
||||
# Remaining MLA projections (registered on this module).
|
||||
self.fused_qkv_a_proj = DeepSeekV2FusedQkvAProjLinear(
|
||||
hidden_size,
|
||||
[q_lora_rank, kv_lora_rank + qk_rope_head_dim],
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkv_a_proj",
|
||||
)
|
||||
self.q_a_layernorm = RMSNorm(q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
q_lora_rank,
|
||||
num_heads * qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(kv_lora_rank, eps=config.rms_norm_eps)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
num_heads * v_head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
|
||||
)
|
||||
|
||||
def _should_prepare_mqa_query(
|
||||
self,
|
||||
attn_metadata: "MLACommonMetadata | None",
|
||||
cudagraph_runtime_mode: CUDAGraphMode,
|
||||
) -> bool:
|
||||
return (
|
||||
attn_metadata is None
|
||||
or not self._use_sparse_mha(attn_metadata)
|
||||
or cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_c, k_pe = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
kw = self.indexer.wk_weights_proj(hidden_states)[0]
|
||||
index_k = kw[:, : self.indexer.head_dim]
|
||||
index_weights = kw[:, self.indexer.head_dim :]
|
||||
else:
|
||||
index_k = None
|
||||
index_weights = None
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
output = torch.empty(
|
||||
(num_tokens, self.num_local_heads * self.v_head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
forward_context = get_forward_context()
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
indexer_slot = (
|
||||
slot_mapping.get(self.indexer.k_cache.prefix)
|
||||
if self.indexer is not None
|
||||
else None
|
||||
)
|
||||
hisparse_cache = self.hisparse_cache
|
||||
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
|
||||
self.impl.prepare_for_batch(layer_attn_metadata)
|
||||
|
||||
prepare_mqa_query = self._should_prepare_mqa_query(
|
||||
layer_attn_metadata, forward_context.cudagraph_runtime_mode
|
||||
)
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
has_indexer = True
|
||||
indexer_k_norm_w = self.indexer.k_norm.weight
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
indexer_k_norm_eps = self.indexer.k_norm.eps
|
||||
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
||||
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
|
||||
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
|
||||
index_k_out = torch.empty_like(index_k) if self.use_pcp else None
|
||||
indexer_softmax_scale = self.indexer.softmax_scale
|
||||
indexer_n_head_scale = self.indexer.n_head**-0.5
|
||||
else:
|
||||
has_indexer = False
|
||||
indexer_k_norm_w = None
|
||||
indexer_k_norm_bias = None
|
||||
indexer_k_norm_eps = 1e-6
|
||||
indexer_k_rope_cos_sin_cache = None
|
||||
indexer_k_cache = None
|
||||
indexer_cache_shuffled = False
|
||||
index_k_out = None
|
||||
indexer_softmax_scale = 0.0
|
||||
indexer_n_head_scale = 0.0
|
||||
|
||||
if forward_context.attn_metadata is None or self.use_pcp:
|
||||
mla_kv_cache = None
|
||||
mla_k_scale = None
|
||||
indexer_k_cache = None
|
||||
mla_slot = None
|
||||
indexer_slot = None
|
||||
else:
|
||||
mla_kv_cache = None if hisparse_cache is not None else self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
if self.use_pcp or hisparse_cache is None:
|
||||
kv_c_out = torch.empty_like(kv_c)
|
||||
k_pe_out = torch.empty_like(k_pe)
|
||||
else:
|
||||
kv_c_out = torch.empty_like(kv_c)
|
||||
k_pe_out = torch.empty_like(k_pe)
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
self.q_a_layernorm.weight,
|
||||
self.q_a_layernorm.variance_epsilon,
|
||||
kv_c,
|
||||
self.kv_a_layernorm.weight,
|
||||
self.kv_a_layernorm.variance_epsilon,
|
||||
k_pe,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
index_k,
|
||||
indexer_k_norm_w,
|
||||
indexer_k_norm_bias,
|
||||
indexer_k_norm_eps,
|
||||
indexer_k_rope_cos_sin_cache,
|
||||
self.topk_indices_buffer,
|
||||
slot_mapping=mla_slot,
|
||||
indexer_slot_mapping=indexer_slot,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
indexer_cache_shuffled=indexer_cache_shuffled,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=self.kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
kv_c_out=kv_c_out,
|
||||
k_pe_out=k_pe_out,
|
||||
index_k_out=index_k_out,
|
||||
)
|
||||
|
||||
if hisparse_cache is not None and mla_slot is not None:
|
||||
assert kv_c_out is not None and k_pe_out is not None
|
||||
self.update_kv_cache(
|
||||
kv_c_out,
|
||||
k_pe_out,
|
||||
self.kv_cache,
|
||||
mla_slot,
|
||||
layer_attn_metadata,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
if prepare_mqa_query:
|
||||
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
else:
|
||||
ql_nope = q_nope
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
|
||||
else:
|
||||
index_q = None
|
||||
|
||||
index_q_fp8, index_weights_out, mqa_q = fused_q(
|
||||
positions,
|
||||
q_pe,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
index_q,
|
||||
self.indexer_rope_emb.cos_sin_cache if has_indexer else None,
|
||||
ql_nope,
|
||||
self._q_scale,
|
||||
index_weights,
|
||||
indexer_softmax_scale,
|
||||
indexer_n_head_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
quantize_mqa=self._fp8_query and prepare_mqa_query,
|
||||
)
|
||||
|
||||
self._sparse_indexer_and_attn(
|
||||
positions,
|
||||
q_c,
|
||||
q_nope,
|
||||
q_pe,
|
||||
index_q_fp8,
|
||||
index_k_out,
|
||||
index_weights_out,
|
||||
kv_c_out,
|
||||
k_pe_out,
|
||||
ql_nope,
|
||||
mqa_q,
|
||||
output,
|
||||
)
|
||||
return self.o_proj(output)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _sparse_indexer_and_attn(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_nope: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
index_q_fp8: torch.Tensor | None,
|
||||
index_k: torch.Tensor | None,
|
||||
index_weights_out: torch.Tensor | None,
|
||||
kv_c: torch.Tensor | None,
|
||||
k_pe: torch.Tensor | None,
|
||||
ql_nope: torch.Tensor,
|
||||
mqa_q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
assert index_q_fp8 is not None
|
||||
assert index_weights_out is not None
|
||||
if self.use_pcp:
|
||||
assert index_k is not None
|
||||
sparse_attn_indexer(
|
||||
q_c,
|
||||
self.indexer.k_cache.prefix,
|
||||
self.indexer.k_cache.kv_cache,
|
||||
index_q_fp8,
|
||||
None,
|
||||
index_k,
|
||||
index_weights_out,
|
||||
self.indexer.quant_block_size,
|
||||
self.indexer.scale_fmt,
|
||||
self.indexer.topk_tokens,
|
||||
self.indexer.head_dim,
|
||||
self.indexer.max_model_len,
|
||||
self.indexer.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=not self.use_pcp,
|
||||
use_pcp=self.use_pcp,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name,
|
||||
dcp_rank=(
|
||||
self.dcp_manager.group.rank_in_group
|
||||
if self.dcp_manager is not None
|
||||
else 0
|
||||
),
|
||||
dcp_world_size=(
|
||||
self._vllm_config.parallel_config.decode_context_parallel_size
|
||||
),
|
||||
cp_kv_cache_interleave_size=(
|
||||
self._vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
),
|
||||
skip_topk_buffer_clear=True,
|
||||
topk_backend=self.indexer.indexer_op.topk_backend,
|
||||
)
|
||||
self.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
self.layer_name
|
||||
)
|
||||
if attn_metadata is None:
|
||||
output.zero_()
|
||||
return
|
||||
attn_metadata = cast("MLACommonMetadata", attn_metadata)
|
||||
self.impl.prepare_for_batch(attn_metadata)
|
||||
|
||||
if self.use_pcp:
|
||||
assert kv_c is not None and k_pe is not None
|
||||
kv_for_cache, kpe_for_cache, cache_slot_mapping = (
|
||||
maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c,
|
||||
k_pe.unsqueeze(1),
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens,
|
||||
True,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_for_cache,
|
||||
kpe_for_cache,
|
||||
kv_cache,
|
||||
cache_slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
if num_actual == 0:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
if self._use_sparse_mha(attn_metadata):
|
||||
assert kv_c is not None and k_pe is not None
|
||||
mha_q_pe = (
|
||||
self.rotary_emb(positions, q_pe)[0]
|
||||
if mqa_q.dtype != q_pe.dtype
|
||||
else mqa_q
|
||||
)
|
||||
mha_q = torch.cat((q_nope, mha_q_pe), dim=-1)
|
||||
self.forward_impl(
|
||||
mha_q,
|
||||
kv_c,
|
||||
k_pe.unsqueeze(1),
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
return
|
||||
|
||||
if self._fp8_kv_needs_view:
|
||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||
if self._fp8_query:
|
||||
# FlashInfer sparse: single packed fp8 query.
|
||||
mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[
|
||||
:num_actual
|
||||
]
|
||||
else:
|
||||
mqa_q_arg = (ql_nope[:num_actual], mqa_q[:num_actual])
|
||||
|
||||
if self.use_pcp and self.impl.dcp_world_size > self.impl.pcp_world_size:
|
||||
if isinstance(mqa_q_arg, tuple):
|
||||
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
|
||||
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
|
||||
elif not self.use_pcp and self.impl.dcp_world_size > 1:
|
||||
assert self.dcp_manager is not None
|
||||
if isinstance(mqa_q_arg, tuple):
|
||||
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
|
||||
assert self.dcp_manager.query_gather is not None
|
||||
mqa_q_arg = self.dcp_manager.query_gather(mqa_q_arg)
|
||||
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
mqa_q_arg, kv_cache, attn_metadata, self
|
||||
)
|
||||
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None and self.dcp_manager is not None
|
||||
seq_lens: torch.Tensor | None
|
||||
query_start_loc: torch.Tensor | None
|
||||
if self.use_pcp:
|
||||
decode_metadata = getattr(attn_metadata, "decode", None)
|
||||
if decode_metadata is not None:
|
||||
seq_lens = decode_metadata.seq_lens
|
||||
else:
|
||||
all_seq_lens = cast(
|
||||
torch.Tensor,
|
||||
attn_metadata.seq_lens, # type: ignore[attr-defined]
|
||||
)
|
||||
seq_lens = all_seq_lens[: attn_metadata.num_decodes]
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
else:
|
||||
# The backend emits (0, -inf) for empty local shards, so no
|
||||
# PCP-only empty-shard metadata is needed.
|
||||
seq_lens = None
|
||||
query_start_loc = None
|
||||
# Under PCP+DCP the prefill rows attended over the gathered KV, so
|
||||
# only the decode rows carry an LSE and take part in the merge.
|
||||
num_merge_rows = lse.shape[0]
|
||||
if num_merge_rows == attn_out.shape[0]:
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
elif num_merge_rows > 0:
|
||||
attn_out[:num_merge_rows] = self.dcp_manager.combine(
|
||||
attn_out[:num_merge_rows],
|
||||
lse,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
# NOTE(woosuk): While the below does not need to be in the eager region,
|
||||
# we put it here to avoid copying the attention output. Move this back to the
|
||||
# captured region once forward_mqa supports `out` argument.
|
||||
x = attn_out.view(
|
||||
num_actual, self.num_local_heads, self.kv_lora_rank
|
||||
).transpose(0, 1)
|
||||
out = (
|
||||
output[:num_actual]
|
||||
.view(num_actual, self.num_local_heads, self.v_head_dim)
|
||||
.transpose(0, 1)
|
||||
)
|
||||
torch.bmm(x, self.W_UV, out=out)
|
||||
if self.use_pcp and num_actual < output.shape[0]:
|
||||
output[num_actual:].zero_()
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,199 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
Backend = Literal["cute", "dsv3_fused_a"]
|
||||
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GLM52ProjectionSpec:
|
||||
n: int
|
||||
k: int
|
||||
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...]
|
||||
dsv3_tokens: frozenset[int] = frozenset()
|
||||
|
||||
def build_plan(self) -> dict[int, ResolvedCall]:
|
||||
plan: dict[int, ResolvedCall] = {
|
||||
num_tokens: ("cute", config) for num_tokens, config in self.cute_configs
|
||||
}
|
||||
plan.update(
|
||||
(num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens
|
||||
)
|
||||
return plan
|
||||
|
||||
|
||||
GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec(
|
||||
n=2624,
|
||||
k=6144,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 128, 4, static_k=6144)),
|
||||
(2, SkinnyGemmConfig(2, 128, 2)),
|
||||
),
|
||||
dsv3_tokens=frozenset(range(3, 17)),
|
||||
)
|
||||
|
||||
GLM52_Q_B_PROJECTION = GLM52ProjectionSpec(
|
||||
n=2048,
|
||||
k=2048,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 128, 4, static_k=2048)),
|
||||
(2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)),
|
||||
),
|
||||
dsv3_tokens=frozenset(range(3, 17)),
|
||||
)
|
||||
|
||||
# The MTP eh_proj is a plain nn.Linear, so it gets its plan through
|
||||
# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4.
|
||||
GLM52_EH_PROJECTION = GLM52ProjectionSpec(
|
||||
n=6144,
|
||||
k=12288,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)),
|
||||
(2, SkinnyGemmConfig(2, 64, 2)),
|
||||
(3, SkinnyGemmConfig(3, 64, 2)),
|
||||
),
|
||||
)
|
||||
|
||||
GLM52_PROJECTIONS = {
|
||||
(spec.n, spec.k): spec
|
||||
for spec in (
|
||||
GLM52_QKV_A_PROJECTION,
|
||||
GLM52_Q_B_PROJECTION,
|
||||
GLM52_EH_PROJECTION,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _is_sm103() -> bool:
|
||||
return current_platform.is_device_capability((10, 3))
|
||||
|
||||
|
||||
def _is_supported_row_major(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
||||
|
||||
|
||||
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and _is_supported_row_major(x)
|
||||
and _is_supported_row_major(weight)
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and x.is_cuda
|
||||
and weight.is_cuda
|
||||
and x.device == weight.device
|
||||
and x.shape[1] == weight.shape[1]
|
||||
)
|
||||
|
||||
|
||||
def run_glm52_plan(
|
||||
plan: dict[int, ResolvedCall] | None,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
if plan is None or not _runtime_ok(x, weight):
|
||||
return None
|
||||
entry = plan.get(x.shape[0])
|
||||
if entry is None:
|
||||
return None
|
||||
|
||||
backend, config = entry
|
||||
if backend == "cute":
|
||||
if not shape_dynamic_skinny_gemm.is_available():
|
||||
return None
|
||||
return shape_dynamic_skinny_gemm(x, weight, config)
|
||||
|
||||
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
||||
return None
|
||||
output = torch.empty(
|
||||
(x.shape[0], weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
|
||||
return output
|
||||
|
||||
|
||||
def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None:
|
||||
if configs and shape_dynamic_skinny_gemm.is_available():
|
||||
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs)
|
||||
|
||||
|
||||
def build_glm52_plan(
|
||||
weight: torch.Tensor | None, dtype: torch.dtype
|
||||
) -> dict[int, ResolvedCall] | None:
|
||||
"""Plan for a weight the walk below cannot reach (a plain ``nn.Linear``)."""
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return None
|
||||
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
|
||||
return None
|
||||
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
|
||||
if spec is None:
|
||||
return None
|
||||
_request_warmup(dtype, {config for _, config in spec.cute_configs})
|
||||
return spec.build_plan()
|
||||
|
||||
|
||||
class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod):
|
||||
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
|
||||
super().__init__()
|
||||
self._plan = plan
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if bias is None:
|
||||
output = run_glm52_plan(self._plan, x, layer.weight)
|
||||
if output is not None:
|
||||
return output
|
||||
return super().apply(layer, x, bias)
|
||||
|
||||
|
||||
def enable_glm52_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return
|
||||
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
for child in module.modules():
|
||||
if (
|
||||
not isinstance(child, LinearBase)
|
||||
or type(child.quant_method) is not UnquantizedLinearMethod
|
||||
):
|
||||
continue
|
||||
weight = getattr(child, "weight", None)
|
||||
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
|
||||
continue
|
||||
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
|
||||
if spec is None:
|
||||
continue
|
||||
child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan())
|
||||
warmup_configs.update(config for _, config in spec.cute_configs)
|
||||
|
||||
_request_warmup(dtype, warmup_configs)
|
||||
@@ -1,454 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
fused_embed_norm,
|
||||
has_full_vocab_on_rank,
|
||||
make_input_embedding,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2ForCausalLM,
|
||||
DeepseekV2MLP,
|
||||
DeepseekV2MoE,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
get_pp_missing_layer_names,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
)
|
||||
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
|
||||
|
||||
|
||||
class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str,
|
||||
config=None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
if config is None:
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
|
||||
layer_idx = int(prefix.split(sep=".")[-1])
|
||||
self.layer_idx = layer_idx
|
||||
self.use_mha = False
|
||||
self.use_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
|
||||
self.self_attn = DeepseekV32Attention(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
if (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % moe_layer_freq == 0
|
||||
):
|
||||
# Keep the MoE output un-reduced. Non-SP fuses its all-reduce into
|
||||
# the next layer norm; SP keeps the token shard local.
|
||||
self.mlp = DeepseekV2MoE(
|
||||
config=config,
|
||||
parallel_config=parallel_config,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.mlp",
|
||||
apply_routed_scale_to_output=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = DeepseekV2MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
is_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
attn_in: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
full_num_tokens = positions.shape[0]
|
||||
|
||||
if residual is None:
|
||||
# First layer: hidden_states is the embedding (the residual).
|
||||
residual = hidden_states
|
||||
# ``attn_in`` is input_layernorm(embedding) already computed fused
|
||||
# with the embedding gather; otherwise apply it here.
|
||||
hidden_states = (
|
||||
attn_in if attn_in is not None else self.input_layernorm(hidden_states)
|
||||
)
|
||||
elif self.use_sequence_parallel:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
else:
|
||||
# The previous layer's MLP/MoE output is left un-reduced; fuse its
|
||||
# all-reduce into this input_layernorm.
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.input_layernorm
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
|
||||
# self_attn's o_proj runs reduce_results=False; reduce before RMSNorm.
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_reduce_scatter(hidden_states)
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
|
||||
if self.use_sequence_parallel and isinstance(self.mlp, DeepseekV2MoE):
|
||||
hidden_states = self.mlp(hidden_states, already_sequence_parallel=True)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class DeepseekV32Model(torch.nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
self.device = current_platform.device_type
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
# DSA is always sparse (has index_topk); allocate the shared top-k
|
||||
# buffer the indexer writes and the sparse MLA backend reads.
|
||||
self.is_v32 = True
|
||||
# On the model, not a local: the MTP proposer shares it with the draft.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(
|
||||
self.topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = make_input_embedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
# The fused embed+norm gather needs the full table on-rank.
|
||||
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV32DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
self.aux_hidden_state_layers = tuple[int, ...]()
|
||||
self.num_redundant_experts = parallel_config.eplb_config.num_redundant_experts
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
attn_in = None
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
elif self.replicated_embed:
|
||||
assert input_ids is not None
|
||||
# Full table on-rank: gather the embedding and the first layer's
|
||||
# input_layernorm in one launch. ``attn_in`` is the pre-normed
|
||||
# attention input; ``hidden_states`` is the (residual) embedding.
|
||||
hidden_states, attn_in = fused_embed_norm(
|
||||
input_ids,
|
||||
self.embed_tokens.weight,
|
||||
chain_weight=self.layers[self.start_layer].input_layernorm.weight,
|
||||
eps=self.config.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
assert input_ids is not None
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
if attn_in is not None:
|
||||
attn_in = sp_shard(attn_in)
|
||||
assert residual is None, "Currently, SP is not supported with PP"
|
||||
|
||||
aux_hidden_states = []
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual, attn_in)
|
||||
attn_in = None
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
assert not self.use_sequence_parallel, (
|
||||
"Currently, SP is not supported with PP"
|
||||
)
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
packed_hidden_states = torch.cat(
|
||||
[hidden_states, *aux_hidden_states], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)
|
||||
packed_hidden_states = packed_hidden_states[:full_num_tokens]
|
||||
hidden_states, *aux_hidden_states = packed_hidden_states.split(
|
||||
hidden_size, dim=-1
|
||||
)
|
||||
else:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
else:
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.norm
|
||||
)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# DSA-only: MLA (fused_qkv_a_proj) + the fused indexer wk/weights_proj +
|
||||
# routed experts. No MHA (qkv_proj) or ROCm shared-expert-fusion paths.
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("fused_qkv_a_proj", "q_a_proj", 0),
|
||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||
("wk_weights_proj", "wk", 0),
|
||||
("wk_weights_proj", "weights_proj", 1),
|
||||
]
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# MTP / nextn layers are loaded by the MTP model, not here.
|
||||
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
|
||||
continue
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# Experts are handled below; skip here before the name rewrite.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
name = name_mapped
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
|
||||
"""DSA causal LM — DeepSeek V2/V3 orchestration with the DSA backbone.
|
||||
|
||||
Serves DeepSeek V3.2 and any architecture reusing DSA (e.g. GLM-5.2).
|
||||
"""
|
||||
|
||||
model_cls = DeepseekV32Model
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
if self.config.model_type == "glm_moe_dsa":
|
||||
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
# Same as the base, but keyed on the MoE block type rather than the
|
||||
# decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module).
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.moe_layers = []
|
||||
self.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if isinstance(layer.mlp, DeepseekV2MoE):
|
||||
example_moe = layer.mlp
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
@@ -1,494 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
fused_embed_eh_norm,
|
||||
has_full_vocab_on_rank,
|
||||
make_input_embedding,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2MixtureOfExperts,
|
||||
DeepseekV2MoE,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm52_low_latency_gemm import (
|
||||
build_glm52_plan,
|
||||
enable_glm52_low_latency_gemm,
|
||||
run_glm52_plan,
|
||||
)
|
||||
from .model import DeepseekV32DecoderLayer
|
||||
|
||||
|
||||
class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
self._eh_plan = (
|
||||
build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype)
|
||||
if config.model_type == "glm_moe_dsa"
|
||||
else None
|
||||
)
|
||||
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.mtp_block = DeepseekV32DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
config=config,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
embed_table: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Fused zero pos-0 + enorm(embeds) + hnorm(prev) + cat -> [N, 2H]. With a
|
||||
# replicated table the caller passes ``embed_table`` so the embedding
|
||||
# lookup is folded in too (fused_embed_eh_norm); otherwise the embeds are
|
||||
# precomputed and go through the model-local fused_eh_norm.
|
||||
if embed_table is not None:
|
||||
eh_input = fused_embed_eh_norm(
|
||||
positions,
|
||||
input_ids,
|
||||
embed_table,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
assert inputs_embeds is not None
|
||||
eh_input = fused_eh_norm(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
is_sequence_parallel = self.mtp_block.use_sequence_parallel
|
||||
if is_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, eh_input
|
||||
)
|
||||
eh_input = sp_shard(eh_input)
|
||||
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
|
||||
if hidden_states is None:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
# Recycle the POST-final-norm hidden into the next draft step. The
|
||||
# residual-add is fused into the final RMSNorm so it is computed
|
||||
# exactly once, and the result is returned for both tuple positions:
|
||||
# the draft-logits hidden (compute_logits applies the LM head only) and
|
||||
# the recycled previous_hidden_states. Recycling the pre-final-norm
|
||||
# hidden mismatches the draft model's hnorm and lowers MTP acceptance;
|
||||
# post-norm recycle matches deepseek_mtp.py (PR #45895). The tuple form
|
||||
# is understood by both the V2 speculator (isinstance-tuple check) and
|
||||
# the legacy proposer (model_returns_tuple is True for the
|
||||
# DeepSeekMTPModel architecture).
|
||||
if is_sequence_parallel:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
|
||||
else:
|
||||
# The MoE output is left un-reduced; fuse its all-reduce into the
|
||||
# final norm, as the main model does at layer boundaries.
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.shared_head.norm
|
||||
)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepseekV32MultiTokenPredictorLayer(
|
||||
vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = make_input_embedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
|
||||
)
|
||||
# A full on-rank table lets the eh_norm fusion fold in the embedding gather.
|
||||
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
# index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse.
|
||||
for layer in self.layers.values():
|
||||
self_attn = getattr(layer.mtp_block, "self_attn", None)
|
||||
if self_attn is not None and hasattr(self_attn, "skip_topk"):
|
||||
self_attn.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, slot_ids: torch.Tensor):
|
||||
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
|
||||
num_slots = slot_ids.numel()
|
||||
for layer in self.layers.values():
|
||||
self_attn = getattr(layer.mtp_block, "self_attn", None)
|
||||
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
|
||||
topk_indices_buffer = self_attn.topk_indices_buffer
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# With a replicated table, defer the embedding gather to fused_eh_norm
|
||||
# (folded into the enorm/hnorm/cat launch); otherwise gather it here.
|
||||
embed_table = None
|
||||
if inputs_embeds is None:
|
||||
if self.replicated_embed:
|
||||
embed_table = self.embed_tokens.weight
|
||||
else:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
embed_table,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# hidden_states is already post-final-norm (produced in the layer
|
||||
# forward and recycled as-is); apply the LM head only, without a
|
||||
# second RMSNorm.
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Greedy draft token ids via per-rank argmax over the vocab shard.
|
||||
|
||||
Saves the full-vocab all-gather ``compute_logits`` does; same tokens.
|
||||
Name is fixed by the protocol the proposer probes for
|
||||
(``use_local_argmax_reduction``).
|
||||
"""
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, hidden_states
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = DeepseekV32MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if self.config.model_type == "glm_moe_dsa":
|
||||
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
self.set_moe_parameters()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], self.config.hidden_size
|
||||
)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
self.num_expert_groups = self.config.n_group
|
||||
self.moe_layers = []
|
||||
self.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers.values():
|
||||
mlp = layer.mtp_block.mlp
|
||||
if isinstance(mlp, DeepseekV2MoE):
|
||||
example_moe = mlp
|
||||
self.moe_mlp_layers.append(mlp)
|
||||
self.moe_layers.append(mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""See ``DeepseekV32MultiTokenPredictor.get_top_tokens``."""
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return (
|
||||
get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
and "embed_tokens" not in name
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("fused_qkv_a_proj", "q_a_proj", 0),
|
||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||
("wk_weights_proj", "wk", 0),
|
||||
("wk_weights_proj", "weights_proj", 1),
|
||||
]
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
# A tied top-level embed_tokens has no spec layer to rewrite
|
||||
# from; the draft needs its own copy under PP.
|
||||
param = params_dict.get(name) if "embed_tokens" in name else None
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
else:
|
||||
name = name_mapped
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
num_chunks = 1
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = chunk_name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if (
|
||||
spec_layer != self.model.mtp_start_layer_idx
|
||||
and ".layers" not in name
|
||||
):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint."
|
||||
)
|
||||
return loaded_params
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,555 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
|
||||
|
||||
from vllm.cute_utils import _TORCH_TO_CUTE_DTYPE, cvt
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _make_fake_tensor(dtype, shape, divisibility):
|
||||
stride = tuple(
|
||||
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
|
||||
for i in range(len(shape))
|
||||
)
|
||||
return cute.runtime.make_fake_tensor(
|
||||
dtype,
|
||||
shape,
|
||||
stride,
|
||||
assumed_align=divisibility * dtype.width // 8,
|
||||
)
|
||||
|
||||
|
||||
def is_fused_q_cutedsl_supported(
|
||||
q_pe: torch.Tensor,
|
||||
index_q: torch.Tensor | None,
|
||||
ql_nope: torch.Tensor,
|
||||
*,
|
||||
has_indexer: bool,
|
||||
quantize_mqa: bool,
|
||||
) -> bool:
|
||||
if not (
|
||||
current_platform.has_device_capability(100)
|
||||
and q_pe.dtype == torch.bfloat16
|
||||
and q_pe.shape[-1] == 64
|
||||
and (
|
||||
not quantize_mqa
|
||||
or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512)
|
||||
)
|
||||
# One warp per head group; a non-multiple would trip the kernel's own
|
||||
# assert instead of falling back to Triton.
|
||||
and q_pe.shape[1] % 4 == 0
|
||||
):
|
||||
return False
|
||||
return not has_indexer or (
|
||||
index_q is not None
|
||||
and index_q.dtype == torch.bfloat16
|
||||
and index_q.shape[1] % 16 == 0
|
||||
and index_q.shape[-1] == 128
|
||||
)
|
||||
|
||||
|
||||
def fused_q_cutedsl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
rope_cache: torch.Tensor,
|
||||
ql_nope: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
mqa_output: torch.Tensor,
|
||||
idx_q: torch.Tensor,
|
||||
idx_rope_cache: torch.Tensor,
|
||||
idx_weights: torch.Tensor,
|
||||
idx_weights_softmax_scale: float,
|
||||
idx_weights_head_scale: float,
|
||||
idx_q_fp8: torch.Tensor,
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = True,
|
||||
quantize_mqa: bool = True,
|
||||
) -> None:
|
||||
_, num_heads, rope_dim = q_pe.shape
|
||||
nope_dim = ql_nope.shape[-1] if quantize_mqa else 0
|
||||
_, num_idx_heads, idx_dim = idx_q.shape
|
||||
|
||||
if has_indexer:
|
||||
idx_rope_type = _TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
|
||||
idx_weights_type = _TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
|
||||
else:
|
||||
idx_dim = num_idx_heads = 0
|
||||
idx_q = idx_rope_cache = idx_q_fp8 = None
|
||||
idx_weights = idx_weights_out = None
|
||||
idx_rope_type = idx_weights_type = None
|
||||
|
||||
rope_type = _TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
|
||||
compiled = FusedQKernel.compile(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
rope_type,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
idx_rope_type,
|
||||
idx_weights_type,
|
||||
index_rope_interleave,
|
||||
)
|
||||
compiled(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope if quantize_mqa else None,
|
||||
q_scale.view(1) if quantize_mqa else None,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
float(idx_weights_softmax_scale * idx_weights_head_scale),
|
||||
)
|
||||
|
||||
|
||||
class FusedQKernel:
|
||||
def __init__(
|
||||
self,
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
assert rope_dim == 64
|
||||
assert nope_dim in (0, 512)
|
||||
assert idx_dim in (128, 0)
|
||||
|
||||
self.rope_dim = rope_dim
|
||||
self.nope_dim = nope_dim
|
||||
self.num_heads = num_heads
|
||||
self.idx_dim = idx_dim
|
||||
self.num_idx_heads = num_idx_heads
|
||||
self.index_rope_interleave = index_rope_interleave
|
||||
|
||||
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
|
||||
# indexer: rope_dim=64, nope_dim=64, num_heads=32
|
||||
|
||||
self.num_warps = 4
|
||||
assert num_heads % self.num_warps == 0
|
||||
assert num_idx_heads % (4 * self.num_warps) == 0
|
||||
self.num_ctas_per_tok = num_heads // self.num_warps
|
||||
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
stream: CUstream,
|
||||
):
|
||||
num_tokens = positions.shape[0]
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
grid = (num_tokens, self.num_ctas_per_tok, 1)
|
||||
else:
|
||||
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
|
||||
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
|
||||
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
|
||||
|
||||
self.kernel(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
).launch(
|
||||
grid=grid,
|
||||
block=(self.num_warps * 32, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
):
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
token_id, group_id, _ = cute.arch.block_idx()
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
token_id,
|
||||
group_id,
|
||||
)
|
||||
else:
|
||||
# CTA-specialization
|
||||
bid, _, _ = cute.arch.block_idx()
|
||||
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
|
||||
if bid < num_mqa_ctas:
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
bid // self.num_ctas_per_tok,
|
||||
bid % self.num_ctas_per_tok,
|
||||
)
|
||||
else:
|
||||
bid -= num_mqa_ctas
|
||||
self.indexer(
|
||||
positions,
|
||||
idx_q,
|
||||
idx_q_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
bid // self.num_ctas_per_idx_tok,
|
||||
bid % self.num_ctas_per_idx_tok,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def mqa(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
head_id = group_id * self.num_warps + warp_id
|
||||
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
# no need to do Q calculation for MHA
|
||||
if cutlass.const_expr(self.nope_dim == 0):
|
||||
q0, q1 = cvt.bf16x2_to_fp32x2(
|
||||
cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id]
|
||||
)
|
||||
cos = q_pe_rope_cache[pos, lane_id].to(Float32)
|
||||
sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32)
|
||||
result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin)
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result
|
||||
return
|
||||
inv_scale = 1.0 / q_scale[0]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
|
||||
|
||||
##### issue all loads asap #####
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
|
||||
|
||||
src_ql_nope = cute.local_tile(
|
||||
ql_nope[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
|
||||
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
|
||||
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
|
||||
|
||||
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
|
||||
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
|
||||
|
||||
##### process NoPE #####
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
|
||||
dst_Q_nope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
|
||||
|
||||
##### process RoPE ######
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rCos = rCos_raw.to(Float32)
|
||||
rSin = rSin_raw.to(Float32)
|
||||
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
|
||||
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
|
||||
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
|
||||
dst_Q_rope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
|
||||
)
|
||||
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
|
||||
|
||||
@cute.jit
|
||||
def indexer(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
subwarp_id = tid // 8
|
||||
sublane_id = tid % 8
|
||||
head_id = group_id * (4 * self.num_warps) + subwarp_id
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
|
||||
##### issue all loads first #####
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
src_idx_q_rope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
|
||||
else:
|
||||
src_idx_q_rope = cute.zipped_divide(
|
||||
idx_q[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 0 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
|
||||
)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 8 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
|
||||
)
|
||||
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
src_idx_q_nope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
|
||||
|
||||
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
|
||||
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
|
||||
# unpack to FP32
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
rCos = rCos_raw
|
||||
rSin = rSin_raw
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
|
||||
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
|
||||
|
||||
# apply rope
|
||||
for i in cutlass.range_constexpr(4):
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
|
||||
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
|
||||
rQ_rope_f32[i * 2 + 0] = r0
|
||||
rQ_rope_f32[i * 2 + 1] = r1
|
||||
else:
|
||||
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
|
||||
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
|
||||
rQ_rope_f32[0 + i] = r0
|
||||
rQ_rope_f32[4 + i] = r1
|
||||
|
||||
# amax
|
||||
amax = Float32(1e-4)
|
||||
for i in cutlass.range_constexpr(8):
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
|
||||
|
||||
# warp reduction among 8 lanes
|
||||
for i in cutlass.range_constexpr(3):
|
||||
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
|
||||
amax = cute.arch.fmax(amax, other)
|
||||
|
||||
# compute scale from amax
|
||||
# exp2(ceil(log2(scale))) via bit manipulation
|
||||
scale = amax * (1.0 / 448.0)
|
||||
bits = scale.bitcast(Uint32)
|
||||
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
|
||||
scale = exp_bits.bitcast(Float32)
|
||||
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
|
||||
|
||||
for i in cutlass.range_constexpr(8):
|
||||
rQ_nope_f32[i] *= inv_scale
|
||||
rQ_rope_f32[i] *= inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# quantize and store
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
|
||||
dst_idx_q_nope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
|
||||
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
dst_idx_q_rope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
|
||||
else:
|
||||
dst_idx_q_rope = cute.zipped_divide(
|
||||
idx_q_fp8[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (0,)),
|
||||
dst_idx_q_rope[None, 0 + sublane_id],
|
||||
)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (1,)),
|
||||
dst_idx_q_rope[None, 8 + sublane_id],
|
||||
)
|
||||
|
||||
# scale indexer weights
|
||||
if sublane_id == 0:
|
||||
w = idx_weights[token_id, head_id].to(Float32)
|
||||
idx_weights_out[token_id, head_id] = w * scale * weight_scale
|
||||
|
||||
@staticmethod
|
||||
@cache
|
||||
def compile(
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
rope_type: type[cutlass.Numeric],
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
idx_rope_type: type[cutlass.Numeric] | None,
|
||||
idx_weights_type: type[cutlass.Numeric] | None,
|
||||
index_rope_interleave: bool,
|
||||
):
|
||||
num_tokens = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
|
||||
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
|
||||
q_pe = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
|
||||
)
|
||||
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
|
||||
ql_nope = (
|
||||
_make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
|
||||
)
|
||||
if nope_dim
|
||||
else None
|
||||
)
|
||||
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None
|
||||
mqa_output = _make_fake_tensor(
|
||||
Float8E4M3FN if nope_dim else BFloat16,
|
||||
(num_tokens, num_heads, nope_dim + rope_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
|
||||
if idx_rope_type is not None:
|
||||
index_q = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_rope_cache = _make_fake_tensor(
|
||||
idx_rope_type, (max_pos, rope_dim), divisibility=8
|
||||
)
|
||||
index_weights = _make_fake_tensor(
|
||||
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
|
||||
)
|
||||
index_q_fp8 = _make_fake_tensor(
|
||||
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_weights_out = _make_fake_tensor(
|
||||
Float32, (num_tokens, num_idx_heads), divisibility=4
|
||||
)
|
||||
else:
|
||||
index_q = index_rope_cache = index_q_fp8 = None
|
||||
index_weights = index_weights_out = None
|
||||
|
||||
kernel = FusedQKernel(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
index_rope_interleave,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
index_q,
|
||||
index_rope_cache,
|
||||
index_weights,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
Float32(0.0),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4 model — hardware-isolated entry point.
|
||||
|
||||
The actual implementation lives under ``nvidia/`` and ``amd/``; this module
|
||||
picks the right one for the current platform and re-exports the public
|
||||
classes used by the model registry and quantization config lookup.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quant_config import DeepseekV4FP8Config
|
||||
|
||||
# Pick the per-platform implementation. The NVIDIA branch is the static
|
||||
# default that mypy sees; the ROCm/XPU branches override at runtime and are
|
||||
# kept type-compatible via ``# type: ignore[assignment]``.
|
||||
if current_platform.is_rocm():
|
||||
from .amd.dspark import ( # type: ignore[assignment]
|
||||
DSparkDeepseekV4ForCausalLM,
|
||||
)
|
||||
from .amd.model import DeepseekV4ForCausalLM
|
||||
from .amd.mtp import DeepSeekV4MTP
|
||||
from .common.vl_model import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
from .vl_stub import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
from .xpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .xpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .xpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
elif current_platform.is_cpu():
|
||||
from .cpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .cpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .cpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
from .vl_stub import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
else:
|
||||
from .common.vl_model import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.dspark import ( # type: ignore[assignment]
|
||||
DSparkDeepseekV4ForCausalLM,
|
||||
)
|
||||
from .nvidia.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .nvidia.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
|
||||
__all__ = [
|
||||
"DSparkDeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTP",
|
||||
"DeepseekV4FP8Config",
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
]
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,514 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark draft model for DeepSeek-V4 on ROCm/AMD (gfx950).
|
||||
|
||||
ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
|
||||
``amd/mtp.py``:
|
||||
|
||||
* import ``DeepseekV4DecoderLayer`` from the AMD ``.model`` (aiter/triton
|
||||
attention + MHC CustomOp path) instead of the nvidia one;
|
||||
* route the MHC head through the ``HCHeadOp`` CustomOp dispatcher (aiter /
|
||||
tilelang / triton / torch) instead of calling the tilelang kernels directly,
|
||||
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
|
||||
or TileLang fused MHC is available; False only on the torch fallback);
|
||||
* use the AITER MegaMoE path (``aiter_mega_moe``) instead of the DeepGEMM
|
||||
mega-MoE path.
|
||||
|
||||
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
|
||||
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
|
||||
pure torch / Triton and shared with the nvidia implementation unchanged.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from .mega_moe import finalize_mega_moe_weights
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
|
||||
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DSparkDeepseekV4Model(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_eps = config.hc_eps
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
|
||||
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
|
||||
|
||||
# Shared with the target (aliased by the speculator's loading utility).
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
self.main_proj = ReplicatedLinear(
|
||||
config.hidden_size * len(self.target_layer_ids),
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "main_proj"),
|
||||
)
|
||||
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4DecoderLayer(
|
||||
current_vllm_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
|
||||
)
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov + confidence heads
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", True):
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
config.hidden_size + config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
)
|
||||
|
||||
# MHC head CustomOp dispatcher (aiter / tilelang / triton / torch),
|
||||
# replacing the direct nvidia tilelang kernel call.
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
|
||||
|
||||
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
|
||||
"""
|
||||
return self.main_norm(self.main_proj(aux_hidden_states))
|
||||
|
||||
@torch.inference_mode()
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
main_x: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Insert the sliding-window context KV for every draft layer.
|
||||
|
||||
Mirrors the reference DSparkAttention: each layer derives its context KV
|
||||
from the SAME projected target hidden ``main_x``, via that layer's own
|
||||
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
|
||||
layer's context slots.
|
||||
|
||||
``context_slot_mappings`` is a per-layer list (each entry is the context
|
||||
slot mapping for that layer's kv-cache group, since the hybrid manager may
|
||||
place draft layers in different groups). ``None`` (or a ``None`` entry)
|
||||
runs the projection to reserve workspace but writes nothing (profiling).
|
||||
"""
|
||||
for i, layer in enumerate(self.layers):
|
||||
slot_mapping = (
|
||||
None if context_slot_mappings is None else context_slot_mappings[i]
|
||||
)
|
||||
attn = layer.attn
|
||||
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
|
||||
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
|
||||
qr_kv, _ = attn.fused_wqa_wkv(main_x)
|
||||
kv = qr_kv[..., attn.q_lora_rank :]
|
||||
kv = attn.kv_norm(kv)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
_insert_context_kv(attn, kv, context_positions, slot_mapping)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
|
||||
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
|
||||
residual = post_mix = res_mix = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
# On the fused-MHC path the trailing hc_post must be applied here; on the
|
||||
# aiter unfused path (ROCm default) the decoder layer already applied
|
||||
# hc_post in-layer and returned None mixes, so this is skipped. Mirrors
|
||||
# amd/mtp.py.
|
||||
last_layer = self.layers[-1]
|
||||
if last_layer.use_fused_mhc:
|
||||
hidden_states = last_layer.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
# hc_head reduces the hc copies; return the PRE-norm head hidden.
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
self.hc_head_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _insert_context_kv(
|
||||
attn: nn.Module,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
|
||||
|
||||
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
|
||||
query and discard it, since context tokens have no query). Mirrors
|
||||
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
|
||||
"""
|
||||
swa_cache = attn.swa_cache_layer.kv_cache
|
||||
block_size = attn.swa_cache_layer.block_size
|
||||
cos_sin_cache = attn.rotary_emb.cos_sin_cache
|
||||
cache_dtype = swa_cache.dtype
|
||||
n_ctx = kv.shape[0]
|
||||
dummy_q = torch.zeros(
|
||||
(n_ctx, attn.n_local_heads, attn.head_dim),
|
||||
dtype=kv.dtype,
|
||||
device=kv.device,
|
||||
)
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged layout (the gfx950 aiter SWA cache path).
|
||||
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.padded_heads,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
elif cache_dtype == torch.bfloat16:
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
else: # per-tensor fp8 (torch.float8_e4m3fn)
|
||||
# NOTE(rocm): unreachable on ROCm/aiter, where the SWA cache dtype is
|
||||
# uint8 (fp8_ds_mla) or bfloat16. This branch relies on FlashInfer-only
|
||||
# attributes (``_flashinfer_fp8_*``) that the aiter attention layer does
|
||||
# not define; kept for parity with the nvidia path.
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
dummy_q_fp8,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn._flashinfer_fp8_kv_scale,
|
||||
attn._flashinfer_fp8_q_scale_inv,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
has_own_lm_head = False
|
||||
# Full-vocab draft: draft ids are target ids, no remapping needed.
|
||||
draft_id_to_target_id = None
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
self.model = DSparkDeepseekV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
# Shared with the target (aliased by the speculator's load utility).
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.combine_hidden_states(aux_hidden_states)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
|
||||
# layer, named by its prefix.
|
||||
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
|
||||
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
self.model.precompute_and_store_context_kv(
|
||||
context_states, context_positions, context_slot_mappings
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Returns the pre-norm hc_head hidden ([T, hidden_size]).
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Base logits U_k = lm_head(norm(head_hidden))."""
|
||||
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Full-vocab draft: base logits, no d2t scatter.
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids # full-vocab: draft ids are target ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
if self.model.confidence_head is None:
|
||||
raise RuntimeError(
|
||||
"compute_confidence() requires a confidence head, but the "
|
||||
"checkpoint did not provide confidence_head weights."
|
||||
)
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
# --- Weight loading ----------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
|
||||
|
||||
Non-mtp weights (embed/head/main layers) belong to the target model and
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
routed_experts_prefix=(
|
||||
"" if self.model.layers[0].ffn.use_mega_moe else "routed_experts"
|
||||
),
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
if "confidence_head." in name:
|
||||
loaded_confidence_head = True
|
||||
|
||||
# ``.scale`` -> per-method scale suffix.
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
|
||||
# E8M0 expert scales: keep raw exponent bytes.
|
||||
if ".experts." in name:
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
success = param.weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
|
||||
# Stacked rules only apply to decoder-layer weights. Head-stack params
|
||||
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
|
||||
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
finalize_mega_moe_weights(self)
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
Returns None for non-mtp weights (owned by the target model).
|
||||
"""
|
||||
m = re.match(r"mtp\.(\d+)\.(.*)", name)
|
||||
if m is None:
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
|
||||
return None
|
||||
# Head-stack params live at model level (mtp.last), context combiner at
|
||||
# model level (mtp.0); everything else is a per-layer decoder block.
|
||||
head_prefixes = (
|
||||
"norm.",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
"markov_head.",
|
||||
"confidence_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
):
|
||||
return f"model.{rest}"
|
||||
return f"model.layers.{stage}.{rest}"
|
||||
@@ -1,301 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_dequantize,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _append_shared_expert_kernel(
|
||||
topk_ids_ptr,
|
||||
topk_weights_ptr,
|
||||
out_ids_ptr,
|
||||
out_weights_ptr,
|
||||
num_tokens,
|
||||
shared_expert_id,
|
||||
TOPK: tl.constexpr,
|
||||
ROUTED_PER_RANK: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
mask = rows < num_tokens
|
||||
for k in tl.static_range(TOPK):
|
||||
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
|
||||
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
|
||||
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
|
||||
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
|
||||
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
|
||||
tl.store(
|
||||
out_ids_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.zeros_like(rows) + shared_expert_id,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
out_weights_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.full([BLOCK_M], 1.0, tl.float32),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
|
||||
# instance and passes its own weights per call.
|
||||
_runtime_cache: dict[tuple, Any] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
swiglu_limit: float | None,
|
||||
fuse_shared_expert: bool = False,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
ep_group = get_ep_group()
|
||||
self.ep_rank = ep_group.rank_in_group
|
||||
self.ep_size = ep_group.world_size
|
||||
self.num_routed_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
|
||||
# A fused shared expert is one extra local expert per rank and one
|
||||
# extra route per token, to the token's own rank.
|
||||
self.fuse_shared_expert = fuse_shared_expert
|
||||
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
|
||||
self.num_experts = self.num_local_experts * self.ep_size
|
||||
self.routed_top_k = top_k
|
||||
self.top_k = top_k + fuse_shared_expert
|
||||
self.shared_expert_id = (
|
||||
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.swiglu_limit = swiglu_limit or 0.0
|
||||
self.max_num_tokens = next_power_of_2(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def make_param(rows: int, cols: int) -> nn.Parameter:
|
||||
param = nn.Parameter(
|
||||
torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(param, {"weight_loader": self.weight_loader})
|
||||
return param
|
||||
|
||||
self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2)
|
||||
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
|
||||
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
|
||||
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
|
||||
# (max_tok_per_rank, runtime), ascending by capacity.
|
||||
self._runtimes: list[tuple[int, Any]] = []
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_id = expert_id - self.experts_start_idx
|
||||
if not 0 <= local_expert_id < self.num_routed_local_experts:
|
||||
return False if return_success else None
|
||||
expert_data = param.data[local_expert_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
|
||||
expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size)
|
||||
elif shard_id != "w2":
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
expert_data.copy_(loaded_weight.view(torch.uint8))
|
||||
return True if return_success else None
|
||||
|
||||
def load_shared_expert(self, shared_experts: nn.Module) -> None:
|
||||
slot = self.num_routed_local_experts
|
||||
for linear, weight, weight_scale in (
|
||||
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
|
||||
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
|
||||
):
|
||||
block_scale = get_fp8_block_weight_scale(linear)
|
||||
assert block_scale is not None, "shared expert must be block-FP8"
|
||||
group = GroupShape(
|
||||
linear.weight.shape[0] // block_scale.shape[0],
|
||||
linear.weight.shape[1] // block_scale.shape[1],
|
||||
)
|
||||
packed, scale = mxfp4_quantize(
|
||||
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
|
||||
)
|
||||
weight.data[slot].copy_(packed.view(torch.uint8))
|
||||
weight_scale.data[slot].copy_(scale.view(torch.uint8))
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._runtimes:
|
||||
return
|
||||
n = self.num_local_experts
|
||||
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
self.w13_weight.data, 16, True
|
||||
)
|
||||
self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True
|
||||
)
|
||||
self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
self.w2_weight.data, 16, False
|
||||
)
|
||||
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
|
||||
)
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
FIXED_SLOT_MAX_MTPR,
|
||||
)
|
||||
|
||||
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
|
||||
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
|
||||
# must also be a power of two.
|
||||
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
|
||||
capacities = [self.max_num_tokens]
|
||||
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
|
||||
fixed_slot_max_tokens
|
||||
):
|
||||
capacities.insert(0, fixed_slot_max_tokens)
|
||||
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
|
||||
|
||||
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
build_mega_moe_bundle_plan,
|
||||
)
|
||||
|
||||
return build_mega_moe_bundle_plan(
|
||||
max_tok_per_rank,
|
||||
experts_per_rank=self.num_local_experts,
|
||||
model_dim=self.hidden_size,
|
||||
inter_dim=self.intermediate_size,
|
||||
world_size=self.ep_size,
|
||||
).fixed_slot_dispatch
|
||||
|
||||
def get_runtime(self, max_tok_per_rank: int) -> Any:
|
||||
key = (
|
||||
id(get_ep_group().cpu_group),
|
||||
torch.accelerator.current_device_index(),
|
||||
self.num_experts,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
max_tok_per_rank,
|
||||
self.swiglu_limit,
|
||||
)
|
||||
runtime = self._runtime_cache.get(key)
|
||||
if runtime is None:
|
||||
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
|
||||
|
||||
runtime = MegaMoEV2(
|
||||
rank=self.ep_rank,
|
||||
world_size=self.ep_size,
|
||||
model_dim=self.hidden_size,
|
||||
inter_dim=self.intermediate_size,
|
||||
experts=self.num_experts,
|
||||
topk=self.top_k,
|
||||
quant="a8w4",
|
||||
w1=self.w13_weight,
|
||||
w1_scale=self.w13_weight_scale,
|
||||
w2=self.w2_weight,
|
||||
w2_scale=self.w2_weight_scale,
|
||||
max_tok_per_rank=max_tok_per_rank,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
)
|
||||
self._runtime_cache[key] = runtime
|
||||
return runtime
|
||||
|
||||
def select_runtime(self, num_tokens: int) -> Any:
|
||||
# Dispatch is collective, so every EP rank must pick the same runtime.
|
||||
if is_forward_context_available():
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is not None:
|
||||
num_tokens = max(
|
||||
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
|
||||
)
|
||||
for max_tok_per_rank, runtime in self._runtimes:
|
||||
if num_tokens <= max_tok_per_rank:
|
||||
return runtime
|
||||
raise ValueError(
|
||||
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
|
||||
)
|
||||
|
||||
def append_shared_expert(
|
||||
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Remap routed ids to the per-rank (routed + shared) layout and append
|
||||
a weight-1.0 route to this rank's shared slot."""
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out_ids = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
|
||||
)
|
||||
out_weights = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
|
||||
)
|
||||
if num_tokens:
|
||||
block_m = 64
|
||||
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
|
||||
topk_ids.contiguous(),
|
||||
topk_weights.to(torch.float32).contiguous(),
|
||||
out_ids,
|
||||
out_weights,
|
||||
num_tokens,
|
||||
self.shared_expert_id,
|
||||
TOPK=self.routed_top_k,
|
||||
ROUTED_PER_RANK=self.num_routed_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
)
|
||||
return out_weights, out_ids
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert self._runtimes
|
||||
runtime = self.select_runtime(hidden_states.shape[0])
|
||||
if self.fuse_shared_expert:
|
||||
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
|
||||
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
||||
runtime._s1_w1 = self.w13_weight
|
||||
runtime._s1_w1_scale = self.w13_weight_scale
|
||||
runtime.w2 = self.w2_weight
|
||||
runtime.w2_scale = self.w2_weight_scale
|
||||
return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids)
|
||||
|
||||
|
||||
DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def finalize_mega_moe_weights(model: nn.Module) -> None:
|
||||
for module in list(model.modules()):
|
||||
experts = getattr(module, "experts", None)
|
||||
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
|
||||
if experts.fuse_shared_expert:
|
||||
experts.load_shared_expert(module.shared_experts)
|
||||
module.shared_experts = None
|
||||
experts.finalize_weights()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,558 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
|
||||
|
||||
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
|
||||
pieces that have no analogue in V3/V32:
|
||||
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
|
||||
the fused ``eh_proj``);
|
||||
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
|
||||
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
|
||||
* V4-specific checkpoint weight-name remapping in ``load_weights``.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import align_mtp_quant_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .mega_moe import finalize_mega_moe_weights
|
||||
from .model import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which MoERunner method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
# Other FP8 linear scales (including shared experts) always use
|
||||
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
|
||||
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
prefix: str,
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
|
||||
# fusing them the way V3 does with eh_proj.
|
||||
self.e_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.e_proj",
|
||||
)
|
||||
self.h_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.h_proj",
|
||||
)
|
||||
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.mtp_block = DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
|
||||
# reshape to (T, hc_mult, D) — the training-time layout — before
|
||||
# the fused norm pass so both inputs are 3D-friendly.
|
||||
previous_hidden_states = previous_hidden_states.view(
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight.data,
|
||||
self.hnorm.weight.data,
|
||||
self.enorm.variance_epsilon,
|
||||
self.hc_mult,
|
||||
)
|
||||
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=input_ids
|
||||
)
|
||||
if self.mtp_block.use_fused_mhc:
|
||||
hidden_states = self.mtp_block.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
# Return the flat pre-hc_head residual so it can be re-fed as the
|
||||
# next spec step's `previous_hidden_states` when
|
||||
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
|
||||
return hidden_states.flatten(1)
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.device = current_platform.device_type
|
||||
|
||||
topk_tokens = config.index_topk
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
# Three aux streams shared across all MTP layers, mirroring
|
||||
# DeepseekV4Model. ROCm runs the same work serially for now.
|
||||
aux_stream_list = (
|
||||
None
|
||||
if current_platform.is_rocm()
|
||||
else [torch.cuda.Stream() for _ in range(3)]
|
||||
)
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepSeekV4MultiTokenPredictorLayer(
|
||||
vllm_config,
|
||||
self.topk_indices_buffer,
|
||||
f"{prefix}.layers.{idx}",
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def _pre_head(
|
||||
self,
|
||||
mtp_layer: DeepSeekV4MultiTokenPredictorLayer,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
|
||||
# hc_head here so logits are computed from the dense hidden state.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
|
||||
)
|
||||
hidden_states = mtp_layer.hc_head_op(
|
||||
hidden_states,
|
||||
mtp_layer.hc_head_fn,
|
||||
mtp_layer.hc_head_scale,
|
||||
mtp_layer.hc_head_base,
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
return _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
logits = self.logits_processor(
|
||||
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
|
||||
)
|
||||
return logits
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# Vocab-parallel argmax for the greedy draft: per-rank head projection
|
||||
# + local argmax + a [batch, 2*tp] (value, index) reduce, instead of
|
||||
# all-gathering the full vocab logits.
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
# The draft head reuses DeepseekV4DecoderLayer, so its fused linears must
|
||||
# unfold to the same checkpoint shards for per-layer quant config lookup.
|
||||
packed_modules_mapping = DeepseekV4ForCausalLM.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
align_mtp_quant_config(self.quant_config, self.config)
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Greedy-draft path used when use_local_argmax_reduction is enabled:
|
||||
# vocab-parallel argmax, no full-vocab logits.
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
WEIGHT_NAME_REMAPPING: dict[str, str] = {
|
||||
".emb.tok_emb.weight": ".embed_tokens.weight",
|
||||
".head.weight": ".shared_head.head.weight",
|
||||
".norm.weight": ".shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def _remap_weight_name(name: str) -> str:
|
||||
"""Remap checkpoint weight names to model parameter names."""
|
||||
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
|
||||
if old_pattern in name:
|
||||
name = name.replace(old_pattern, new_pattern)
|
||||
return name
|
||||
|
||||
def _find_mtp_layer_idx(name: str) -> int:
|
||||
subnames = name.split(".")
|
||||
for subname in subnames:
|
||||
try:
|
||||
# we return the first encountered integer
|
||||
return int(subname)
|
||||
except ValueError:
|
||||
continue
|
||||
return 0
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_scale_name(name: str) -> str:
|
||||
# Quark checkpoints and QuarkW8A8Fp8PerBlock use ``.weight_scale``.
|
||||
# Native block-FP8 layers register ``.weight_scale_inv``. Rename
|
||||
# to ``_inv`` only when that variant exists and the plain one
|
||||
# does not.
|
||||
if name.endswith(".weight_scale") and name not in params_dict:
|
||||
inv = name.removesuffix(".weight_scale") + ".weight_scale_inv"
|
||||
if inv in params_dict:
|
||||
return inv
|
||||
# Otherwise leave the name unchanged: either it already matches a
|
||||
# param, or it is genuinely unknown and should surface the normal
|
||||
# KeyError downstream rather than be silently rewritten.
|
||||
return name
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
routed_experts_prefix=(
|
||||
"" if first_layer.mtp_block.ffn.use_mega_moe else "routed_experts"
|
||||
),
|
||||
)
|
||||
|
||||
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
|
||||
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
|
||||
# for the rename below based on the model's expert dtype.
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer_idx = _find_mtp_layer_idx(name)
|
||||
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
|
||||
# `model.layers.{num_hidden_layers + i}.*` so that
|
||||
# get_spec_layer_idx_from_weight_name can identify them.
|
||||
name = name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
name = _remap_weight_name(name)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
|
||||
continue
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
name = _resolve_scale_name(name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# Reinterpret E8M0 scales as uint8 to preserve raw
|
||||
# exponent bytes; numeric copy_() would zero them.
|
||||
# Mirrors the main DeepseekV4 loader.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(".bias", ".e_score_correction_bias")
|
||||
name = _resolve_scale_name(name)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may have "
|
||||
f"been quantized without including the MTP layers. "
|
||||
f"Use a checkpoint that includes MTP layer weights, "
|
||||
f"or disable speculative decoding."
|
||||
)
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
finalize_mega_moe_weights(self)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"h_proj",
|
||||
"e_proj",
|
||||
"shared_head",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
# treat rest weights as weights for transformer layer block
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
# treat shared weights as top level weights
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,529 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Multimodal preprocessing for the DeepSeek-V4 vision variants
|
||||
(DeepSeek-V4-Flash-Vision-Exp).
|
||||
|
||||
The image transform and sentinel-block construction are ported from the
|
||||
official repository's ``image_processor.py`` so that token counts bit-match
|
||||
the reference. Each ``<|deepseek_image|>`` placeholder in the prompt expands
|
||||
to a variable-length block of sentinel tokens; only positions with
|
||||
``type == IMAGE`` receive vision embeddings, the other sentinels are looked
|
||||
up from learned embedding vectors in the model.
|
||||
|
||||
Unlike the reference (which uses out-of-vocab ids ``vocab_size + type``),
|
||||
the sentinel block borrows five consecutive reserved tokenizer tokens
|
||||
(``<|place_holder_mm_span_0431|>`` .. ``_0435|>``): they are special tokens
|
||||
the tokenizer never emits from plain text, so the ids stay in-vocabulary and
|
||||
work with stock token validation, logprobs and detokenization. The ids are
|
||||
pure markers — every sentinel position's embedding is overwritten with
|
||||
vision/sentinel vectors before the decoder layers see it, exactly like the
|
||||
reference's out-of-vocab scheme.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, assert_never
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature, DeepseekV4Config
|
||||
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalPromptUpdates,
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
UpdateMode,
|
||||
_plan_prompt_updates,
|
||||
)
|
||||
|
||||
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
|
||||
COMPRESS_PAD_TO = 4
|
||||
|
||||
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
|
||||
|
||||
# Sentinel roles borrow five consecutive ``<|place_holder_mm_span_XXXX|>``
|
||||
# tokens (reserved special tokens, never emitted from plain text). The order
|
||||
# must match IMAGE_START..IMAGE_END above so that
|
||||
# ``id == IMAGE_SENTINEL_BASE_ID + type``.
|
||||
IMAGE_SENTINEL_BASE_ID = 129257
|
||||
IMAGE_SENTINEL_TOKEN_NAMES = (
|
||||
"<|place_holder_mm_span_0431|>", # IMAGE_START
|
||||
"<|place_holder_mm_span_0432|>", # IMAGE_PAD
|
||||
"<|place_holder_mm_span_0433|>", # IMAGE
|
||||
"<|place_holder_mm_span_0434|>", # IMAGE_NEW_LINE
|
||||
"<|place_holder_mm_span_0435|>", # IMAGE_END
|
||||
)
|
||||
|
||||
|
||||
def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Boolean mask for image-block sentinel positions (in-vocab ids)."""
|
||||
return (token_ids >= IMAGE_SENTINEL_BASE_ID) & (
|
||||
token_ids < IMAGE_SENTINEL_BASE_ID + len(IMAGE_SENTINEL_TOKEN_NAMES)
|
||||
)
|
||||
|
||||
|
||||
def validate_image_sentinel_ids(tokenizer) -> None:
|
||||
"""Check the borrowed sentinel ids against the tokenizer."""
|
||||
for i, name in enumerate(IMAGE_SENTINEL_TOKEN_NAMES):
|
||||
token_id = tokenizer.convert_tokens_to_ids(name)
|
||||
if token_id != IMAGE_SENTINEL_BASE_ID + i:
|
||||
raise ValueError(
|
||||
f"Image sentinel token {name!r} has id {token_id}, expected "
|
||||
f"{IMAGE_SENTINEL_BASE_ID + i}; the DeepSeek-V4 vision "
|
||||
"sentinel block requires these consecutive reserved ids."
|
||||
)
|
||||
|
||||
|
||||
def grid_tokens(best_height, best_width, patch_size, downsample_ratio):
|
||||
"""Number of LLM tokens the aligner grid occupies (N-layout, including
|
||||
row/align padding)."""
|
||||
n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
|
||||
n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
|
||||
num_tokens = n_llm_h * (n_llm_w + 1) + 2
|
||||
if n_llm_h % 2 == 1:
|
||||
num_tokens += n_llm_w + 1
|
||||
num_tokens += (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2
|
||||
return n_llm_h, n_llm_w, num_tokens
|
||||
|
||||
|
||||
def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
|
||||
r = height / width
|
||||
max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
|
||||
max_h_float = max_w_float * r
|
||||
if max_w_float < 1.0:
|
||||
max_w = 1
|
||||
max_h = (max_n_token - 2) // (max_w + 1)
|
||||
if max_h % 2 == 1:
|
||||
max_h -= 1
|
||||
best_width = max_w * patch_size * downsample_ratio
|
||||
best_height = max_h * patch_size * downsample_ratio
|
||||
elif max_h_float < 2.0:
|
||||
max_h = 2
|
||||
max_w = ((max_n_token - 2) // max_h) - 1
|
||||
assert max_w > 1
|
||||
best_width = max_w * patch_size * downsample_ratio
|
||||
best_height = max_h * patch_size * downsample_ratio
|
||||
else:
|
||||
max_w = math.floor(max_w_float)
|
||||
max_h = math.floor(max_h_float)
|
||||
if max_h % 2 == 1:
|
||||
max_h -= 1
|
||||
beta = min(
|
||||
max_w * patch_size * downsample_ratio / width,
|
||||
max_h * patch_size * downsample_ratio / height,
|
||||
)
|
||||
best_width = math.floor(width * beta / patch_size) * patch_size
|
||||
best_height = math.floor(height * beta / patch_size) * patch_size
|
||||
n_llm_h, n_llm_w, num_tokens = grid_tokens(
|
||||
best_height, best_width, patch_size, downsample_ratio
|
||||
)
|
||||
return n_llm_h, n_llm_w, best_height, best_width, num_tokens
|
||||
|
||||
|
||||
def safe_resize(
|
||||
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
|
||||
):
|
||||
max_n_token -= COMPRESS_PAD_TO - 1
|
||||
n_llm_h, n_llm_w, num_tokens = grid_tokens(
|
||||
best_height, best_width, patch_size, downsample_ratio
|
||||
)
|
||||
budget = max_n_token
|
||||
while num_tokens > max_n_token:
|
||||
n_llm_h, n_llm_w, best_height, best_width, num_tokens = solve_resize_ratio(
|
||||
height, width, patch_size, downsample_ratio, budget
|
||||
)
|
||||
budget -= 1
|
||||
return n_llm_h, n_llm_w, best_height, best_width
|
||||
|
||||
|
||||
def load_image(
|
||||
image: Image.Image,
|
||||
*,
|
||||
patch_size: int,
|
||||
downsample_ratio: int,
|
||||
max_n_token: int,
|
||||
min_pixels: int,
|
||||
max_wh_ratio: float | None,
|
||||
):
|
||||
"""Transform one PIL image into ViT patches.
|
||||
|
||||
Same math as the reference ``load_image``, except the image is already
|
||||
decoded (vLLM supplies PIL images instead of a record dict).
|
||||
"""
|
||||
p = patch_size
|
||||
image = image.convert("RGB")
|
||||
width, height = image.size
|
||||
if max_wh_ratio is not None and width > height * max_wh_ratio:
|
||||
width = height * max_wh_ratio
|
||||
if 0 < width * height < min_pixels:
|
||||
ratio = (min_pixels / (width * height)) ** 0.5
|
||||
width = int(width * ratio)
|
||||
height = int(height * ratio)
|
||||
best_width = math.ceil(width / p) * p
|
||||
best_height = math.ceil(height / p) * p
|
||||
n_llm_h, n_llm_w, best_height, best_width = safe_resize(
|
||||
height, width, best_height, best_width, p, downsample_ratio, max_n_token
|
||||
)
|
||||
n_vit_h, n_vit_w = best_height // p, best_width // p
|
||||
if max_wh_ratio is not None and image.width >= max_wh_ratio * image.height:
|
||||
image = image.resize((best_width, best_height))
|
||||
else:
|
||||
image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))
|
||||
x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255
|
||||
x = ((x - 0.5) / 0.5).to(torch.bfloat16)
|
||||
patches = (
|
||||
x.reshape(3, n_vit_h, p, n_vit_w, p)
|
||||
.permute(1, 3, 0, 2, 4)
|
||||
.reshape(n_vit_h * n_vit_w, 3, p, p)
|
||||
)
|
||||
return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w
|
||||
|
||||
|
||||
def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
|
||||
"""Builds the N-layout token types (final order) and the aligner-row order
|
||||
for IMAGE slots."""
|
||||
compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
|
||||
pad_h = n_llm_h % 2
|
||||
rows = n_llm_h + pad_h
|
||||
row_len = n_llm_w + 1
|
||||
pad_last = rows // 2 * row_len % 2 * 2
|
||||
types = torch.tensor(
|
||||
([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h
|
||||
+ [IMAGE_PAD] * (row_len * pad_h),
|
||||
dtype=torch.int64,
|
||||
)
|
||||
order = torch.arange(rows * row_len).view(rows // 2, 2, row_len)
|
||||
order = order.transpose(1, 2).reshape(-1)
|
||||
image_idx = torch.full((rows * row_len,), -1, dtype=torch.int64)
|
||||
image_idx.view(rows, row_len)[:n_llm_h, :n_llm_w] = torch.arange(
|
||||
n_llm_h * n_llm_w
|
||||
).view(n_llm_h, n_llm_w)
|
||||
perm = image_idx[order]
|
||||
perm = perm[perm >= 0]
|
||||
types = torch.cat(
|
||||
[
|
||||
torch.full((compress_pad,), IMAGE_PAD, dtype=torch.int64),
|
||||
torch.tensor([IMAGE_START]),
|
||||
types[order],
|
||||
torch.full((pad_last,), IMAGE_PAD, dtype=torch.int64),
|
||||
torch.tensor([IMAGE_END]),
|
||||
]
|
||||
)
|
||||
return types, perm
|
||||
|
||||
|
||||
def build_image_block_pad_free(n_llm_h: int, n_llm_w: int):
|
||||
"""``build_image_block`` without the position-dependent compressor pad.
|
||||
|
||||
``start_pos`` is chosen so that ``compress_pad == 0``; the pad is instead
|
||||
prepended when the block is spliced into the final prompt, where its
|
||||
position is known.
|
||||
"""
|
||||
return build_image_block(n_llm_h, n_llm_w, COMPRESS_PAD_TO - 1)
|
||||
|
||||
|
||||
class DeepseekV4VLImageProcessor:
|
||||
"""Per-image transform (the PIL-input equivalent of the reference
|
||||
``load_image``)."""
|
||||
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
super().__init__()
|
||||
self.patch_size = config.vision_patch_size
|
||||
self.downsample_ratio = config.vision_downsample_ratio
|
||||
self.max_n_token = config.vision_max_n_token
|
||||
self.min_pixels = config.vision_min_pixels
|
||||
self.max_wh_ratio = config.vision_max_wh_ratio
|
||||
|
||||
def __call__(self, image: Image.Image):
|
||||
return load_image(
|
||||
image,
|
||||
patch_size=self.patch_size,
|
||||
downsample_ratio=self.downsample_ratio,
|
||||
max_n_token=self.max_n_token,
|
||||
min_pixels=self.min_pixels,
|
||||
max_wh_ratio=self.max_wh_ratio,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4VLProcessor:
|
||||
"""Minimal stand-in for the HF processor of DeepSeek-V4 vision models.
|
||||
|
||||
The official repository ships image preprocessing as plain functions in
|
||||
``image_processor.py`` (no ``auto_map`` processor), so this class wraps
|
||||
their ports directly and the model loads without ``--trust-remote-code``.
|
||||
|
||||
``__call__`` returns a ``BatchFeature`` with one entry per image
|
||||
(flattened across images):
|
||||
|
||||
- ``patches``: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` bf16 ViT patches.
|
||||
- ``vit_grid``: ``(num_images, 2)`` int64 ``[n_vit_h, n_vit_w]``.
|
||||
- ``llm_grid``: ``(num_images, 2)`` int64 ``[n_llm_h, n_llm_w]``.
|
||||
- ``perm``: concatenated per-image ``(n_llm_h * n_llm_w,)`` int64 index
|
||||
selecting aligner outputs into the final N-layout order.
|
||||
- ``types``: concatenated per-image pad-free sentinel block types;
|
||||
``block ids = IMAGE_SENTINEL_BASE_ID + types``.
|
||||
"""
|
||||
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.image_processor = DeepseekV4VLImageProcessor(config)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | None = None,
|
||||
images: Sequence[Image.Image] | None = None,
|
||||
return_tensors: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> BatchFeature:
|
||||
patches_list = []
|
||||
vit_grid = []
|
||||
llm_grid = []
|
||||
perm_list = []
|
||||
types_list = []
|
||||
for image in images or []:
|
||||
patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = self.image_processor(image)
|
||||
types, perm = build_image_block_pad_free(n_llm_h, n_llm_w)
|
||||
patches_list.append(patches)
|
||||
vit_grid.append((n_vit_h, n_vit_w))
|
||||
llm_grid.append((n_llm_h, n_llm_w))
|
||||
perm_list.append(perm)
|
||||
types_list.append(types)
|
||||
|
||||
if not patches_list:
|
||||
return BatchFeature({})
|
||||
|
||||
return BatchFeature(
|
||||
{
|
||||
"patches": torch.cat(patches_list),
|
||||
"vit_grid": torch.tensor(vit_grid, dtype=torch.int64),
|
||||
"llm_grid": torch.tensor(llm_grid, dtype=torch.int64),
|
||||
"perm": torch.cat(perm_list),
|
||||
"types": torch.cat(types_list),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self) -> DeepseekV4Config:
|
||||
return self.ctx.get_hf_config(DeepseekV4Config)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> DeepseekV4VLProcessor:
|
||||
if kwargs:
|
||||
raise ValueError(f"Unexpected processor kwargs: {sorted(kwargs)}")
|
||||
return DeepseekV4VLProcessor(self.get_hf_config())
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
# ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of
|
||||
# vision_max_n_token for the compressor-alignment pad, so the full
|
||||
# sentinel block is bounded by vision_max_n_token; the margin is kept
|
||||
# in case that reservation changes.
|
||||
return {
|
||||
"image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1,
|
||||
}
|
||||
|
||||
def get_image_placeholder_token_id(self) -> int:
|
||||
token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER)
|
||||
if token_id is None:
|
||||
raise ValueError(f"Token not found in tokenizer: {IMAGE_PLACEHOLDER}")
|
||||
return token_id
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
hf_config = self.get_hf_config()
|
||||
patch_size = hf_config.vision_patch_size
|
||||
downsample_ratio = hf_config.vision_downsample_ratio
|
||||
max_wh_ratio = hf_config.vision_max_wh_ratio
|
||||
# Every aligner row costs an extra IMAGE_NEW_LINE token, so height is
|
||||
# more expensive than width and a square is *not* optimal: search the
|
||||
# LLM grids that fit the budget for the one with the most ViT patches
|
||||
# (ties broken by the longer sentinel block).
|
||||
budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
|
||||
step = patch_size * downsample_ratio
|
||||
best_grid = (1, 1)
|
||||
best_key = (0, 0)
|
||||
for n_llm_h in range(1, budget):
|
||||
if (
|
||||
grid_tokens(n_llm_h * step, step, patch_size, downsample_ratio)[2]
|
||||
> budget
|
||||
):
|
||||
break
|
||||
for n_llm_w in range(1, budget):
|
||||
if max_wh_ratio is not None and n_llm_w > n_llm_h * max_wh_ratio:
|
||||
break
|
||||
num_tokens = grid_tokens(
|
||||
n_llm_h * step, n_llm_w * step, patch_size, downsample_ratio
|
||||
)[2]
|
||||
if num_tokens > budget:
|
||||
break
|
||||
key = (n_llm_h * n_llm_w, num_tokens)
|
||||
if key > best_key:
|
||||
best_key, best_grid = key, (n_llm_h, n_llm_w)
|
||||
n_llm_h, n_llm_w = best_grid
|
||||
return ImageSize(width=n_llm_w * step, height=n_llm_h * step)
|
||||
|
||||
|
||||
class DeepseekV4VLDummyInputsBuilder(
|
||||
BaseDummyInputsBuilder[DeepseekV4VLProcessingInfo]
|
||||
):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return IMAGE_PLACEHOLDER * mm_counts.get("image", 0)
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
size = self.info.get_image_size_with_most_features()
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=size.width,
|
||||
height=size.height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=mm_options.get("image"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class DeepseekV4VLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekV4VLProcessingInfo]
|
||||
):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
vit_grid = hf_inputs.get("vit_grid")
|
||||
llm_grid = hf_inputs.get("llm_grid")
|
||||
|
||||
if vit_grid is None or llm_grid is None:
|
||||
empty = torch.empty(0, dtype=torch.long)
|
||||
patch_sizes = perm_sizes = types_sizes = empty
|
||||
else:
|
||||
patch_sizes = vit_grid.prod(-1)
|
||||
perm_sizes = llm_grid.prod(-1)
|
||||
n_llm_h, n_llm_w = llm_grid[:, 0], llm_grid[:, 1]
|
||||
# Pad-free block length; same formula as ``grid_tokens`` given
|
||||
# the LLM grid.
|
||||
types_sizes = (
|
||||
n_llm_h * (n_llm_w + 1)
|
||||
+ 2
|
||||
+ (n_llm_h % 2) * (n_llm_w + 1)
|
||||
+ (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2
|
||||
)
|
||||
|
||||
return {
|
||||
"patches": MultiModalFieldConfig.flat_from_sizes("image", patch_sizes),
|
||||
"vit_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
"llm_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
"perm": MultiModalFieldConfig.flat_from_sizes("image", perm_sizes),
|
||||
"types": MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", types_sizes, keep_on_cpu=True
|
||||
),
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
image_token_id = self.info.get_image_placeholder_token_id()
|
||||
validate_image_sentinel_ids(self.info.get_tokenizer())
|
||||
image_embed_id = IMAGE_SENTINEL_BASE_ID + IMAGE
|
||||
|
||||
def get_image_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
types: torch.Tensor = out_mm_kwargs["image"][item_idx]["types"].data
|
||||
full = (IMAGE_SENTINEL_BASE_ID + types).tolist()
|
||||
return PromptUpdateDetails.select_token_id(full, image_embed_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=get_image_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
# Same as the base implementation, except that each image block gets
|
||||
# its compressor-alignment pad (``3 - start_pos % 4`` IMAGE_PAD
|
||||
# sentinels) prepended while splicing: the pad depends on the block's
|
||||
# final position in the prompt, which is unknown when the (cacheable)
|
||||
# prompt updates are built.
|
||||
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
|
||||
pad_id = IMAGE_SENTINEL_BASE_ID + IMAGE_PAD
|
||||
|
||||
new_token_ids = list[int]()
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
elif update.mode == UpdateMode.REPLACE:
|
||||
end_idx_to_insert = match.start_idx
|
||||
else:
|
||||
assert_never(update.mode)
|
||||
|
||||
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
|
||||
start_idx = len(new_token_ids)
|
||||
|
||||
tokens = list(update.content.full)
|
||||
if tokens and update.modality == "image":
|
||||
compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO
|
||||
tokens = [pad_id] * compress_pad + tokens
|
||||
|
||||
if tokens:
|
||||
content_is_embed = update.content.is_embed
|
||||
is_embed = (
|
||||
content_is_embed(tokens) if content_is_embed is not None else None
|
||||
)
|
||||
placeholders[update.modality].append(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality=update.modality,
|
||||
item_idx=update.item_idx,
|
||||
start_idx=start_idx,
|
||||
tokens=tokens,
|
||||
is_embed=is_embed,
|
||||
)
|
||||
)
|
||||
new_token_ids.extend(tokens)
|
||||
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
new_token_ids.extend(token_ids[prev_end_idx:])
|
||||
return new_token_ids, result, placeholders
|
||||
@@ -1,23 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
quantize_and_insert_k_cache,
|
||||
)
|
||||
from .fused_indexer_q import MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant
|
||||
from .fused_inv_rope_fp8_quant import fused_inv_rope_fp8_quant
|
||||
|
||||
__all__ = [
|
||||
"MXFP4_BLOCK_SIZE",
|
||||
"build_flashinfer_mixed_sparse_indices",
|
||||
"combine_topk_swa_indices",
|
||||
"compute_global_topk_indices_and_lens",
|
||||
"dequantize_and_gather_k_cache",
|
||||
"fused_indexer_q_rope_quant",
|
||||
"fused_inv_rope_fp8_quant",
|
||||
"quantize_and_insert_k_cache",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,733 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
# MXFP4: 32 elements per block, packed 2 nibbles per byte, ue8m0 block scale.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _get_cos_sin(
|
||||
cos_sin_cache_ptr,
|
||||
cos_sin_cache_stride,
|
||||
pos,
|
||||
HALF_ROT_DIM: tl.constexpr,
|
||||
):
|
||||
block = tl.arange(0, HALF_ROT_DIM)
|
||||
cos = tl.load(cos_sin_cache_ptr + pos * cos_sin_cache_stride + block)
|
||||
cos = cos.to(tl.float32)
|
||||
sin = tl.load(cos_sin_cache_ptr + pos * cos_sin_cache_stride + block + HALF_ROT_DIM)
|
||||
sin = sin.to(tl.float32)
|
||||
return cos, sin
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp32x2_to_fp4x2(x_lo, x_hi):
|
||||
# NOTE: $1 is high nibble, $2 is low nibble
|
||||
return tl.inline_asm_elementwise(
|
||||
"""
|
||||
{
|
||||
.reg .b8 tmp;
|
||||
cvt.rn.satfinite.e2m1x2.f32 tmp, $1, $2;
|
||||
cvt.u32.u8 $0, tmp;
|
||||
}
|
||||
""",
|
||||
constraints="=r,f,f",
|
||||
args=[x_hi, x_lo],
|
||||
dtype=tl.uint32,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
).to(tl.uint8)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_mxfp4_pair(x_lo, x_hi):
|
||||
"""Quantize a block of MXFP4_BLOCK_SIZE fp32 values given as two
|
||||
interleaved halves (x_lo = values at even positions in the block,
|
||||
x_hi = values at odd positions). Returns:
|
||||
- packed : uint8[BLOCK/2] (low nibble = quant(x_lo), high = quant(x_hi))
|
||||
- ue8m0 : scalar uint8 (block scale = 2^(ue8m0 - 127))
|
||||
"""
|
||||
amax = tl.maximum(tl.max(tl.abs(x_lo)), tl.max(tl.abs(x_hi)))
|
||||
# 6 * 2^-126 is from https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/inference/kernel.py#L163
|
||||
amax = tl.maximum(amax, 6.0 * (2**-126))
|
||||
# ue8m0 block scale: 2^ceil(log2(amax/6.0)).
|
||||
log2_ratio = tl.math.ceil(tl.math.log2(amax * (1.0 / 6.0)))
|
||||
log2_ratio = tl.minimum(tl.maximum(log2_ratio, -127.0), 127.0)
|
||||
scale = tl.math.exp2(log2_ratio)
|
||||
ue8m0 = (log2_ratio + 127.0).to(tl.uint8)
|
||||
|
||||
inv_scale = 1.0 / scale
|
||||
packed = _fp32x2_to_fp4x2(x_lo * inv_scale, x_hi * inv_scale)
|
||||
return packed, ue8m0
|
||||
|
||||
|
||||
class FusedIndexerQRopeQuantTritonKernel(
|
||||
VllmTritonJitKernel["FusedIndexerQRopeQuantTritonKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
num_heads: int
|
||||
index_q_half_rot_dim: int
|
||||
index_q_head_dim: int
|
||||
fp8_max: float
|
||||
use_fnuz: bool
|
||||
use_explicit_fma: bool
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# Index Q Quantize
|
||||
index_q_fp8_ptr,
|
||||
index_q_fp8_stride0,
|
||||
index_q_fp8_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
# Index weights
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
FP8_MAX: tl.constexpr = 448.0,
|
||||
USE_FNUZ: tl.constexpr = False,
|
||||
USE_EXPLICIT_FMA: tl.constexpr = False,
|
||||
):
|
||||
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
|
||||
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
|
||||
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
|
||||
# through unchanged.
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
cos, sin = _get_cos_sin(
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
pos,
|
||||
INDEX_Q_HALF_ROT_DIM,
|
||||
)
|
||||
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
|
||||
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
|
||||
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
|
||||
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
|
||||
rot_base = base_ptr + INDEX_Q_NOPE_DIM
|
||||
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
|
||||
if USE_EXPLICIT_FMA:
|
||||
# Match HIP rotary_embedding contraction before bf16 materialization.
|
||||
r_even = tl.fma(x_even, cos, -(x_odd * sin))
|
||||
r_odd = tl.fma(x_odd, cos, x_even * sin)
|
||||
else:
|
||||
r_even = x_even * cos - x_odd * sin
|
||||
r_odd = x_odd * cos + x_even * sin
|
||||
|
||||
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
|
||||
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
|
||||
r_even = r_even.to(tl.bfloat16).to(tl.float32)
|
||||
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
|
||||
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
|
||||
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
|
||||
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
|
||||
|
||||
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
|
||||
# (e4m3fn) elsewhere -- matches the K cache.
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
fp8_base_ptr = (
|
||||
index_q_fp8_ptr
|
||||
+ tok_idx * index_q_fp8_stride0
|
||||
+ head_idx * index_q_fp8_stride1
|
||||
)
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
tl.store(
|
||||
fp8_base_ptr + nope_offset,
|
||||
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2,
|
||||
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2 + 1,
|
||||
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
|
||||
# FP8 weight-fold contract:
|
||||
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
|
||||
# The per-token-per-head q_scale (fp32) IS folded into the output weights
|
||||
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
|
||||
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use
|
||||
# `weights` to apply per-token Q scale inline. See the MXFP4 kernel below for
|
||||
# contrasting convention (scales live with the Q values, weights are NOT
|
||||
# q-scaled).
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
)
|
||||
index_weights = index_weights.to(tl.float32)
|
||||
index_weights *= index_q_scale
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
use_fnuz: bool,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
num_heads=num_heads,
|
||||
index_q_half_rot_dim=rope_dim // 2,
|
||||
index_q_head_dim=head_dim,
|
||||
fp8_max=224.0 if use_fnuz else 448.0,
|
||||
use_fnuz=use_fnuz,
|
||||
use_explicit_fma=current_platform.is_rocm(),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
use_fnuz=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
|
||||
return dict(
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
index_q=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_q_cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.index_q_half_rot_dim * 2),
|
||||
),
|
||||
index_weights=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
index_weights_softmax_scale=1.0,
|
||||
index_weights_head_scale=1.0,
|
||||
index_q_fp8=TritonWarmupTensor(
|
||||
current_platform.fp8_dtype(),
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_weights_out=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
fp8_max=compile_key.fp8_max,
|
||||
use_fnuz=compile_key.use_fnuz,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
index_q_fp8: torch.Tensor,
|
||||
index_weights_out: torch.Tensor,
|
||||
*,
|
||||
fp8_max: float,
|
||||
use_fnuz: bool,
|
||||
) -> LaunchSpec:
|
||||
num_tokens = positions.shape[0]
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
return (num_tokens, num_index_q_heads), dict(
|
||||
pos_ptr=positions,
|
||||
index_q_stride0=index_q.stride(0),
|
||||
index_q_stride1=index_q.stride(1),
|
||||
index_q_cos_sin_ptr=index_q_cos_sin_cache,
|
||||
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
|
||||
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_fp8_stride0=index_q_fp8.stride(0),
|
||||
index_q_fp8_stride1=index_q_fp8.stride(1),
|
||||
INDEX_Q_HEAD_DIM=index_q.shape[2],
|
||||
index_weights_stride=index_weights.stride(0),
|
||||
index_weights_out_stride=index_weights_out.stride(0),
|
||||
FP8_MAX=fp8_max,
|
||||
USE_FNUZ=use_fnuz,
|
||||
USE_EXPLICIT_FMA=current_platform.is_rocm(),
|
||||
enable_fp_fusion=True,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
|
||||
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
|
||||
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
|
||||
(`SparseMQAIndexer`) is enabled."""
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
return (torch.float32, torch.bfloat16)
|
||||
return (torch.float32,)
|
||||
|
||||
|
||||
class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
weights_out_dtype: torch.dtype
|
||||
num_heads: int
|
||||
index_q_half_rot_dim: int
|
||||
index_q_head_dim: int
|
||||
mxfp4_block: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE input (fp/bf16)
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# MXFP4 Q outputs
|
||||
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
|
||||
index_q_mxfp4_stride0,
|
||||
index_q_mxfp4_stride1,
|
||||
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
|
||||
index_q_scale_stride0,
|
||||
index_q_scale_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
MXFP4_BLOCK: tl.constexpr,
|
||||
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
|
||||
# with the Q values in the output scale tensor).
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
):
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
|
||||
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
|
||||
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(MXFP4_BLOCK % 2 == 0)
|
||||
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
|
||||
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
out_base = (
|
||||
index_q_mxfp4_ptr
|
||||
+ tok_idx * index_q_mxfp4_stride0
|
||||
+ head_idx * index_q_mxfp4_stride1
|
||||
)
|
||||
scale_base = (
|
||||
index_q_scale_ptr
|
||||
+ tok_idx * index_q_scale_stride0
|
||||
+ head_idx * index_q_scale_stride1
|
||||
)
|
||||
|
||||
half_off = tl.arange(0, HALF_BLOCK)
|
||||
|
||||
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
|
||||
for b in tl.static_range(NUM_NOPE_BLOCKS):
|
||||
base = b * MXFP4_BLOCK
|
||||
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
|
||||
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
|
||||
tl.store(out_base + base // 2 + half_off, packed)
|
||||
tl.store(scale_base + b, ue8m0)
|
||||
|
||||
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
|
||||
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
|
||||
rot_q_base = q_base + INDEX_Q_NOPE_DIM
|
||||
for b in tl.static_range(NUM_ROPE_BLOCKS):
|
||||
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
|
||||
cos_b = tl.load(
|
||||
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
|
||||
).to(tl.float32)
|
||||
sin_b = tl.load(
|
||||
index_q_cos_sin_ptr
|
||||
+ pos * index_q_cos_sin_stride
|
||||
+ pair_off
|
||||
+ INDEX_Q_HALF_ROT_DIM
|
||||
).to(tl.float32)
|
||||
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
|
||||
r_even = x_even * cos_b - x_odd * sin_b
|
||||
r_odd = x_odd * cos_b + x_even * sin_b
|
||||
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
|
||||
r_even = r_even.to(tl.bfloat16).to(tl.float32)
|
||||
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
|
||||
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
|
||||
tl.store(out_base + rope_byte_off + half_off, packed)
|
||||
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
|
||||
|
||||
# MXFP4 weight-fold contract:
|
||||
# index_weights_out = index_weights * softmax_scale * head_scale
|
||||
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
|
||||
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
|
||||
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
|
||||
# per-block scale is applied by the downstream MXFP4 logits kernel when
|
||||
# dequantizing Q — there is no per-token scalar to fold into `weights`.
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
).to(tl.float32)
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
weights_out_dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
weights_out_dtype=weights_out_dtype,
|
||||
num_heads=num_heads,
|
||||
index_q_half_rot_dim=rope_dim // 2,
|
||||
index_q_head_dim=head_dim,
|
||||
mxfp4_block=MXFP4_BLOCK_SIZE,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
weights_out_dtype=_indexer_weights_out_dtypes(vllm_config),
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
|
||||
packed_head_dim = compile_key.index_q_head_dim // 2
|
||||
scale_head_dim = compile_key.index_q_head_dim // compile_key.mxfp4_block
|
||||
return dict(
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
index_q=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_q_cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.index_q_half_rot_dim * 2),
|
||||
),
|
||||
index_weights=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
index_weights_softmax_scale=1.0,
|
||||
index_weights_head_scale=1.0,
|
||||
index_q_packed=TritonWarmupTensor(
|
||||
torch.uint8,
|
||||
shape=(1, compile_key.num_heads, packed_head_dim),
|
||||
),
|
||||
index_q_scale=TritonWarmupTensor(
|
||||
torch.uint8,
|
||||
shape=(1, compile_key.num_heads, scale_head_dim),
|
||||
),
|
||||
index_weights_out=TritonWarmupTensor(
|
||||
compile_key.weights_out_dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
index_q_packed: torch.Tensor,
|
||||
index_q_scale: torch.Tensor,
|
||||
index_weights_out: torch.Tensor,
|
||||
) -> LaunchSpec:
|
||||
num_tokens = positions.shape[0]
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
return (num_tokens, num_index_q_heads), dict(
|
||||
pos_ptr=positions,
|
||||
index_q_stride0=index_q.stride(0),
|
||||
index_q_stride1=index_q.stride(1),
|
||||
index_q_cos_sin_ptr=index_q_cos_sin_cache,
|
||||
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
|
||||
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_mxfp4_ptr=index_q_packed,
|
||||
index_q_mxfp4_stride0=index_q_packed.stride(0),
|
||||
index_q_mxfp4_stride1=index_q_packed.stride(1),
|
||||
index_q_scale_stride0=index_q_scale.stride(0),
|
||||
index_q_scale_stride1=index_q_scale.stride(1),
|
||||
INDEX_Q_HEAD_DIM=index_q.shape[2],
|
||||
MXFP4_BLOCK=MXFP4_BLOCK_SIZE,
|
||||
index_weights_stride=index_weights.stride(0),
|
||||
index_weights_out_stride=index_weights_out.stride(0),
|
||||
enable_fp_fusion=True,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
def fused_indexer_q_rope_quant(
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
# Index weights
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
use_fp4: bool = False,
|
||||
weights_out_dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[
|
||||
torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""Fused RoPE + quantize Q for the sparse indexer.
|
||||
|
||||
``weights_out_dtype`` is the dtype the downstream scoring kernel takes:
|
||||
fp32 for the dense MQA-logits kernels, bf16 for DeepGEMM's sparse
|
||||
MQA-logits kernels (CUDA MXFP4 path only).
|
||||
|
||||
Weight-fold semantics (important — the two paths differ):
|
||||
|
||||
FP8 path (use_fp4=False, default):
|
||||
q_fp8 : (T, H, HEAD_DIM) platform fp8 (e4m3fnuz on gfx942,
|
||||
e4m3fn elsewhere); per-token-per-head scalar scale
|
||||
(NOT stored — folded into weights below)
|
||||
weights_out = weights * q_scale * softmax_scale * head_scale
|
||||
Rationale: a single per-token q_scale is a scalar the downstream FP8
|
||||
logits kernel would otherwise multiply in. Folding it into `weights`
|
||||
avoids emitting a separate tensor and is free for the logits kernel.
|
||||
|
||||
MXFP4 path (use_fp4=True):
|
||||
q_packed : (T, H, HEAD_DIM // 2) uint8 (2 E2M1 nibbles per byte)
|
||||
q_scale : (T, H, HEAD_DIM // MXFP4_BLOCK_SIZE) uint8 ue8m0 bytes
|
||||
weights_out = weights * softmax_scale * head_scale
|
||||
Rationale: MXFP4 has PER-BLOCK (32-element) scales that live with
|
||||
the Q values — they cannot be folded into a per-token weight
|
||||
scalar, so `weights` carries only the softmax and head scales.
|
||||
|
||||
Returns (q_quant, weights_out) where q_quant is either a Tensor (FP8) or
|
||||
a (values, scales) tuple (MXFP4). This matches the union type accepted
|
||||
by `SparseAttnIndexer.forward_*`.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert index_q.ndim == 3
|
||||
assert index_q_cos_sin_cache.ndim == 2
|
||||
|
||||
num_tokens = positions.shape[0]
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=weights_out_dtype)
|
||||
|
||||
if use_fp4:
|
||||
assert index_q_head_dim % MXFP4_BLOCK_SIZE == 0, (
|
||||
f"head_dim={index_q_head_dim} must be a multiple of MXFP4 block "
|
||||
f"size {MXFP4_BLOCK_SIZE}"
|
||||
)
|
||||
num_scale_blocks = index_q_head_dim // MXFP4_BLOCK_SIZE
|
||||
index_q_packed = torch.empty(
|
||||
(num_tokens, num_index_q_heads, index_q_head_dim // 2),
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
index_q_scale = torch.empty(
|
||||
(num_tokens, num_index_q_heads, num_scale_blocks),
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
if has_cutedsl():
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
|
||||
_INDEXER_Q_MXFP4_KERNEL,
|
||||
)
|
||||
|
||||
_INDEXER_Q_MXFP4_KERNEL(
|
||||
positions=positions,
|
||||
q=index_q,
|
||||
cos_sin_cache=index_q_cos_sin_cache,
|
||||
weights=index_weights,
|
||||
weights_softmax_scale=index_weights_softmax_scale,
|
||||
weights_head_scale=index_weights_head_scale,
|
||||
q_packed=index_q_packed,
|
||||
q_scale=index_q_scale,
|
||||
weights_out=index_weights_out,
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
assert weights_out_dtype == torch.float32, weights_out_dtype
|
||||
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_mxfp4(
|
||||
index_q,
|
||||
positions,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_packed,
|
||||
index_q_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
else:
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL(
|
||||
positions,
|
||||
index_q,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_packed,
|
||||
index_q_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
|
||||
# Values stay uint8 (2 E2M1 nibbles per byte). Scales are 4 ue8m0
|
||||
# bytes per (token, head) reinterpreted as one int32, then squeezed
|
||||
# from (T, H, 1) to (T, H) to match DeepGEMM's expected q_sf rank
|
||||
# (prefill wants 2-D (seq_len, num_heads); decode reshapes this to
|
||||
# 3-D (batch, next_n, num_heads)).
|
||||
return (
|
||||
index_q_packed,
|
||||
index_q_scale.view(torch.int32).squeeze(-1),
|
||||
), index_weights_out
|
||||
|
||||
assert weights_out_dtype == torch.float32, weights_out_dtype
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
use_fnuz = fp8_dtype == torch.float8_e4m3fnuz
|
||||
fp8_max = 224.0 if use_fnuz else 448.0
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=fp8_dtype)
|
||||
if has_cutedsl():
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
|
||||
_INDEXER_Q_FP8_KERNEL,
|
||||
)
|
||||
|
||||
_INDEXER_Q_FP8_KERNEL(
|
||||
positions=positions,
|
||||
q=index_q,
|
||||
cos_sin_cache=index_q_cos_sin_cache,
|
||||
weights=index_weights,
|
||||
weights_softmax_scale=index_weights_softmax_scale,
|
||||
weights_head_scale=index_weights_head_scale,
|
||||
q_fp8=index_q_fp8.view(torch.uint8),
|
||||
weights_out=index_weights_out,
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_fp8(
|
||||
index_q,
|
||||
positions,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
from vllm._custom_ops import fused_indexer_q_rope_quant_cpu
|
||||
|
||||
fused_indexer_q_rope_quant_cpu(
|
||||
positions.contiguous(),
|
||||
index_q.to(torch.float32),
|
||||
index_q_cos_sin_cache.to(torch.float32).contiguous(),
|
||||
index_q_fp8,
|
||||
index_weights.to(torch.float32),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
else:
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
|
||||
positions,
|
||||
index_q,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
fp8_max=fp8_max,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
return index_q_fp8, index_weights_out
|
||||
|
||||
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL = FusedIndexerQRopeQuantTritonKernel()
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL = FusedIndexerQRopeMxFp4TritonKernel()
|
||||
@@ -1,538 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused inverse RoPE + block-scaled FP8 quantization kernel for DeepseekV4 attention.
|
||||
|
||||
Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
|
||||
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
class FusedInvRopeFP8QuantKernel(
|
||||
VllmTritonJitKernel["FusedInvRopeFP8QuantKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
heads_per_group: int
|
||||
fp8_max: float
|
||||
quant_group_size: int
|
||||
chunks_per_head: int
|
||||
nope_dim: int
|
||||
half_rope: int
|
||||
tma_aligned_scales: bool
|
||||
launch_pdl: bool
|
||||
quantize: bool
|
||||
|
||||
@staticmethod
|
||||
# scale_stride_k = align(num_tokens, 4) is only 4-aligned, so its Triton int
|
||||
# class varies with batch size; not specialized so one warmup key covers all
|
||||
# (int64-cast, so perf-neutral).
|
||||
@triton.jit(do_not_specialize=["num_tokens", "scale_stride_k"])
|
||||
def kernel(
|
||||
o_ptr,
|
||||
positions_ptr,
|
||||
cos_sin_cache_ptr,
|
||||
out_ptr,
|
||||
scale_ptr,
|
||||
num_tokens,
|
||||
heads_per_group: tl.constexpr,
|
||||
o_stride_token,
|
||||
o_stride_head,
|
||||
cache_stride_pos,
|
||||
out_stride_group,
|
||||
out_stride_token,
|
||||
scale_stride_group,
|
||||
scale_stride_k,
|
||||
fp8_max: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
QUANT_GROUP_SIZE: tl.constexpr,
|
||||
CHUNKS_PER_HEAD: tl.constexpr,
|
||||
NOPE_DIM: tl.constexpr,
|
||||
HALF_ROPE: tl.constexpr,
|
||||
QUANTIZE: tl.constexpr,
|
||||
TMA_ALIGNED_SCALES: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
# Cast every stride to int64 — without this, Python-int strides are
|
||||
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
|
||||
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
|
||||
pid_token = tl.program_id(0).to(tl.int64)
|
||||
pid_gh = tl.program_id(1).to(tl.int64)
|
||||
o_stride_token = o_stride_token.to(tl.int64)
|
||||
o_stride_head = o_stride_head.to(tl.int64)
|
||||
cache_stride_pos = cache_stride_pos.to(tl.int64)
|
||||
out_stride_group = out_stride_group.to(tl.int64)
|
||||
out_stride_token = out_stride_token.to(tl.int64)
|
||||
scale_stride_group = scale_stride_group.to(tl.int64)
|
||||
scale_stride_k = scale_stride_k.to(tl.int64)
|
||||
|
||||
g = pid_gh // heads_per_group
|
||||
head_in_group = pid_gh % heads_per_group
|
||||
global_head = pid_gh
|
||||
qb_start = head_in_group * CHUNKS_PER_HEAD
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
|
||||
if pid_token >= num_tokens:
|
||||
if not QUANTIZE:
|
||||
return
|
||||
if TMA_ALIGNED_SCALES:
|
||||
packed_offsets = tl.arange(0, CHUNKS_PER_HEAD // 4)
|
||||
scale_addr = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ (head_in_group * (CHUNKS_PER_HEAD // 4) + packed_offsets)
|
||||
* scale_stride_k
|
||||
)
|
||||
tl.store(scale_addr, tl.zeros((CHUNKS_PER_HEAD // 4,), dtype=tl.int32))
|
||||
else:
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
scale_addrs = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ qb_indices * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
|
||||
return
|
||||
|
||||
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
|
||||
|
||||
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
|
||||
offsets = tl.arange(0, HEAD_DIM)
|
||||
x = tl.load(input_base + offsets).to(tl.float32)
|
||||
|
||||
rope_abs_start: tl.constexpr = NOPE_DIM
|
||||
pos = tl.load(positions_ptr + pid_token)
|
||||
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
|
||||
is_rope = offsets >= rope_abs_start
|
||||
rope_local = offsets - rope_abs_start
|
||||
|
||||
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
cs_idx = tl.maximum(rope_local >> 1, 0)
|
||||
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
|
||||
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
|
||||
x_add = x * cos_v + x_partner * sin_v
|
||||
x_sub = x * cos_v - x_partner * sin_v
|
||||
is_even = (rope_local & 1) == 0
|
||||
rotated = tl.where(is_even, x_add, x_sub)
|
||||
x = tl.where(is_rope, rotated, x)
|
||||
|
||||
if not QUANTIZE:
|
||||
out_base = (
|
||||
out_ptr
|
||||
+ g * out_stride_group
|
||||
+ pid_token * out_stride_token
|
||||
+ qb_start * QUANT_GROUP_SIZE
|
||||
)
|
||||
tl.store(out_base + offsets, x)
|
||||
return
|
||||
|
||||
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
|
||||
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
|
||||
scale_raw = block_absmax * (1.0 / fp8_max)
|
||||
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
|
||||
|
||||
scales_exp = tl.reshape(
|
||||
tl.broadcast_to(
|
||||
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
|
||||
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
|
||||
),
|
||||
(HEAD_DIM,),
|
||||
)
|
||||
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
|
||||
|
||||
out_base = (
|
||||
out_ptr
|
||||
+ g * out_stride_group
|
||||
+ pid_token * out_stride_token
|
||||
+ qb_start * QUANT_GROUP_SIZE
|
||||
)
|
||||
tl.store(out_base + offsets, x_quant)
|
||||
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
if TMA_ALIGNED_SCALES:
|
||||
scale_bits = scales.to(tl.int32, bitcast=True)
|
||||
ue8m0_bytes = (scale_bits >> 23) & 0xFF
|
||||
packed_val = tl.sum(
|
||||
tl.reshape(ue8m0_bytes, (CHUNKS_PER_HEAD // 4, 4))
|
||||
<< (tl.arange(0, 4)[None, :] * 8),
|
||||
axis=1,
|
||||
)
|
||||
packed_offsets = tl.arange(0, CHUNKS_PER_HEAD // 4)
|
||||
scale_addr = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ (head_in_group * (CHUNKS_PER_HEAD // 4) + packed_offsets)
|
||||
* scale_stride_k
|
||||
)
|
||||
tl.store(scale_addr, packed_val)
|
||||
else:
|
||||
scale_addrs = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ qb_indices * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addrs, scales)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
heads_per_group: int,
|
||||
head_dim: int,
|
||||
nope_dim: int,
|
||||
rope_dim: int,
|
||||
quant_group_size: int,
|
||||
runtime_fp8_max: float | None = None,
|
||||
runtime_chunks_per_head: int | None = None,
|
||||
runtime_nope_dim: int | None = None,
|
||||
runtime_half_rope: int | None = None,
|
||||
**compile_key_fields: bool,
|
||||
) -> CompileKey:
|
||||
fp8_max = (
|
||||
runtime_fp8_max
|
||||
if runtime_fp8_max is not None
|
||||
else torch.finfo(torch.float8_e4m3fn).max
|
||||
)
|
||||
chunks_per_head = (
|
||||
runtime_chunks_per_head
|
||||
if runtime_chunks_per_head is not None
|
||||
else head_dim // quant_group_size
|
||||
)
|
||||
nope_dim = runtime_nope_dim if runtime_nope_dim is not None else nope_dim
|
||||
half_rope = (
|
||||
runtime_half_rope if runtime_half_rope is not None else rope_dim // 2
|
||||
)
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
heads_per_group=heads_per_group,
|
||||
fp8_max=fp8_max,
|
||||
quant_group_size=quant_group_size,
|
||||
chunks_per_head=chunks_per_head,
|
||||
nope_dim=nope_dim,
|
||||
half_rope=half_rope,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if hf_config is None:
|
||||
return []
|
||||
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
num_heads = int(getattr(hf_config, "num_attention_heads", 0) or 0)
|
||||
num_groups = int(getattr(hf_config, "o_groups", 0) or 0)
|
||||
if num_heads <= 0 or num_groups <= 0:
|
||||
return []
|
||||
|
||||
local_heads = num_heads // tp_size
|
||||
local_groups = num_groups // tp_size
|
||||
if local_groups <= 0:
|
||||
return []
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
return []
|
||||
|
||||
head_dim = int(getattr(hf_config, "head_dim", 512) or 512)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 64) or 64)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
heads_per_group=local_heads // local_groups,
|
||||
head_dim=head_dim,
|
||||
nope_dim=head_dim - rope_dim,
|
||||
rope_dim=rope_dim,
|
||||
quant_group_size=128,
|
||||
tma_aligned_scales=capability.major >= 10,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
quantize=True,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
head_dim = compile_key.chunks_per_head * compile_key.quant_group_size
|
||||
out_dim = compile_key.heads_per_group * head_dim
|
||||
scale_dtype = torch.int32 if compile_key.tma_aligned_scales else torch.float32
|
||||
return dict(
|
||||
o=TritonWarmupTensor(
|
||||
torch.bfloat16,
|
||||
shape=(1, compile_key.heads_per_group, head_dim),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.half_rope * 2),
|
||||
),
|
||||
out_buf=TritonWarmupTensor(
|
||||
torch.float8_e4m3fn if compile_key.quantize else torch.bfloat16,
|
||||
shape=(compile_key.heads_per_group, 1, out_dim),
|
||||
strides=(out_dim, out_dim, 1),
|
||||
),
|
||||
scale_buf=TritonWarmupTensor(
|
||||
scale_dtype,
|
||||
shape=(compile_key.heads_per_group, 1, 16),
|
||||
strides=(16, 16, 16),
|
||||
),
|
||||
num_tokens=1,
|
||||
heads_per_group=compile_key.heads_per_group,
|
||||
quant_group_size=compile_key.quant_group_size,
|
||||
chunks_per_head=compile_key.chunks_per_head,
|
||||
nope_dim=compile_key.nope_dim,
|
||||
half_rope=compile_key.half_rope,
|
||||
tma_aligned_scales=compile_key.tma_aligned_scales,
|
||||
quantize=compile_key.quantize,
|
||||
fp8_max=compile_key.fp8_max,
|
||||
launch_pdl=compile_key.launch_pdl,
|
||||
grid=(1, compile_key.heads_per_group),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
out_buf: torch.Tensor,
|
||||
scale_buf: torch.Tensor,
|
||||
num_tokens: int,
|
||||
*,
|
||||
heads_per_group: int,
|
||||
quant_group_size: int,
|
||||
chunks_per_head: int,
|
||||
nope_dim: int,
|
||||
half_rope: int,
|
||||
tma_aligned_scales: bool,
|
||||
quantize: bool,
|
||||
fp8_max: float,
|
||||
launch_pdl: bool,
|
||||
grid: tuple[int, int],
|
||||
) -> LaunchSpec:
|
||||
return grid, dict(
|
||||
out_ptr=out_buf,
|
||||
scale_ptr=scale_buf,
|
||||
o_stride_token=o.stride(0),
|
||||
o_stride_head=o.stride(1),
|
||||
cache_stride_pos=cos_sin_cache.stride(0),
|
||||
out_stride_group=out_buf.stride(0),
|
||||
out_stride_token=out_buf.stride(1),
|
||||
scale_stride_group=scale_buf.stride(0) if quantize else 0,
|
||||
scale_stride_k=scale_buf.stride(2) if quantize else 0,
|
||||
eps=1e-10,
|
||||
QUANT_GROUP_SIZE=quant_group_size,
|
||||
CHUNKS_PER_HEAD=chunks_per_head,
|
||||
NOPE_DIM=nope_dim,
|
||||
HALF_ROPE=half_rope,
|
||||
QUANTIZE=quantize,
|
||||
TMA_ALIGNED_SCALES=tma_aligned_scales,
|
||||
launch_pdl=launch_pdl,
|
||||
num_stages=1,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
def fused_inv_rope_fp8_quant(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
n_groups: int,
|
||||
heads_per_group: int,
|
||||
nope_dim: int = 448,
|
||||
rope_dim: int = 64,
|
||||
quant_group_size: int = 128,
|
||||
tma_aligned_scales: bool = False,
|
||||
quantize: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused inverse RoPE + block-scaled FP8 quantization.
|
||||
|
||||
Args:
|
||||
o: Attention output [num_tokens, num_heads, head_dim] bf16.
|
||||
positions: Token positions [num_tokens] int64.
|
||||
cos_sin_cache: Precomputed [max_pos, rope_dim] with cos||sin.
|
||||
n_groups: Number of output groups.
|
||||
heads_per_group: Heads per group.
|
||||
nope_dim: Non-RoPE dimensions per head (default 448).
|
||||
rope_dim: RoPE dimensions per head (default 64).
|
||||
quant_group_size: FP8 quantization block size (default 128).
|
||||
tma_aligned_scales: Output INT32 packed UE8M0 for SM100 (True)
|
||||
or FP32 for SM90 (False).
|
||||
quantize: Quantize the rotated output to FP8 and return its scales.
|
||||
|
||||
Returns:
|
||||
Rotated output in [T, G, D] and its FP8 scales. The scale tensor is
|
||||
empty when quantization is disabled.
|
||||
|
||||
"""
|
||||
from vllm.utils.deep_gemm import get_tma_aligned_size
|
||||
|
||||
num_tokens, num_heads, head_dim = o.shape
|
||||
assert num_heads == n_groups * heads_per_group
|
||||
assert head_dim == nope_dim + rope_dim
|
||||
assert head_dim % quant_group_size == 0
|
||||
assert rope_dim % 2 == 0
|
||||
assert cos_sin_cache.shape[-1] == rope_dim
|
||||
assert cos_sin_cache.dtype == torch.float32
|
||||
|
||||
d = heads_per_group * head_dim
|
||||
num_scale_blocks = d // quant_group_size
|
||||
chunks_per_head = head_dim // quant_group_size
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fn
|
||||
fp8_max = torch.finfo(fp8_dtype).max
|
||||
|
||||
tma_aligned_T = get_tma_aligned_size(num_tokens, 4) if quantize else num_tokens
|
||||
if quantize and tma_aligned_scales:
|
||||
assert chunks_per_head % 4 == 0
|
||||
packed_sf_k = (num_scale_blocks + 3) // 4
|
||||
scale_inner = packed_sf_k
|
||||
elif quantize:
|
||||
scale_inner = num_scale_blocks
|
||||
else:
|
||||
scale_inner = 0
|
||||
|
||||
# Run kernel through a custom op so inductor sees an opaque boundary.
|
||||
# It's a pytorch bug, see https://github.com/vllm-project/vllm/issues/41106
|
||||
out_buf, scale_buf = torch.ops.vllm.fused_inv_rope_fp8_quant_kernel(
|
||||
o,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
heads_per_group,
|
||||
quant_group_size,
|
||||
chunks_per_head,
|
||||
nope_dim,
|
||||
rope_dim // 2,
|
||||
tma_aligned_scales,
|
||||
fp8_max,
|
||||
tma_aligned_T,
|
||||
num_tokens,
|
||||
n_groups,
|
||||
d,
|
||||
scale_inner,
|
||||
quantize,
|
||||
)
|
||||
output = out_buf.transpose(0, 1)
|
||||
scales = scale_buf.transpose(0, 1) if quantize else scale_buf
|
||||
return output, scales
|
||||
|
||||
|
||||
def _fused_inv_rope_fp8_quant_kernel_impl(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
heads_per_group: int,
|
||||
quant_group_size: int,
|
||||
chunks_per_head: int,
|
||||
nope_dim: int,
|
||||
half_rope: int,
|
||||
tma_aligned_scales: bool,
|
||||
fp8_max: float,
|
||||
tma_aligned_T: int,
|
||||
num_tokens: int,
|
||||
n_groups: int,
|
||||
d: int,
|
||||
scale_inner: int,
|
||||
quantize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
scale_dtype = torch.int32 if tma_aligned_scales else torch.float32
|
||||
out_buf = torch.empty(
|
||||
(n_groups, num_tokens, d),
|
||||
dtype=torch.float8_e4m3fn if quantize else o.dtype,
|
||||
device=o.device,
|
||||
)
|
||||
if quantize:
|
||||
scale_buf = torch.empty(
|
||||
n_groups * scale_inner * tma_aligned_T,
|
||||
dtype=scale_dtype,
|
||||
device=o.device,
|
||||
).as_strided(
|
||||
(n_groups, num_tokens, scale_inner),
|
||||
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
|
||||
)
|
||||
else:
|
||||
scale_buf = torch.empty(0, dtype=scale_dtype, device=o.device)
|
||||
grid = (tma_aligned_T, n_groups * heads_per_group)
|
||||
launch_pdl = current_platform.is_arch_support_pdl()
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL(
|
||||
o,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
out_buf,
|
||||
scale_buf,
|
||||
num_tokens,
|
||||
heads_per_group=heads_per_group,
|
||||
quant_group_size=quant_group_size,
|
||||
chunks_per_head=chunks_per_head,
|
||||
nope_dim=nope_dim,
|
||||
half_rope=half_rope,
|
||||
tma_aligned_scales=tma_aligned_scales,
|
||||
quantize=quantize,
|
||||
fp8_max=fp8_max,
|
||||
launch_pdl=launch_pdl,
|
||||
grid=grid,
|
||||
)
|
||||
return out_buf, scale_buf
|
||||
|
||||
|
||||
def _fused_inv_rope_fp8_quant_kernel_fake(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
heads_per_group: int,
|
||||
quant_group_size: int,
|
||||
chunks_per_head: int,
|
||||
nope_dim: int,
|
||||
half_rope: int,
|
||||
tma_aligned_scales: bool,
|
||||
fp8_max: float,
|
||||
tma_aligned_T: int,
|
||||
num_tokens: int,
|
||||
n_groups: int,
|
||||
d: int,
|
||||
scale_inner: int,
|
||||
quantize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
scale_dtype = torch.int32 if tma_aligned_scales else torch.float32
|
||||
out_buf = torch.empty(
|
||||
(n_groups, num_tokens, d),
|
||||
dtype=torch.float8_e4m3fn if quantize else o.dtype,
|
||||
device=o.device,
|
||||
)
|
||||
if not quantize:
|
||||
return out_buf, torch.empty(0, dtype=scale_dtype, device=o.device)
|
||||
scale_buf = torch.empty(
|
||||
n_groups * scale_inner * tma_aligned_T,
|
||||
dtype=scale_dtype,
|
||||
device=o.device,
|
||||
).as_strided(
|
||||
(n_groups, num_tokens, scale_inner),
|
||||
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
|
||||
)
|
||||
return out_buf, scale_buf
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_inv_rope_fp8_quant_kernel",
|
||||
op_func=_fused_inv_rope_fp8_quant_kernel_impl,
|
||||
fake_impl=_fused_inv_rope_fp8_quant_kernel_fake,
|
||||
)
|
||||
|
||||
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL = FusedInvRopeFP8QuantKernel()
|
||||
@@ -1,340 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused MTP-input RMSNorm: enorm (with mask-zero at position 0) + hnorm.
|
||||
|
||||
Replaces the eager sequence at the top of the MTP draft forward:
|
||||
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
inputs_embeds = self.enorm(inputs_embeds)
|
||||
previous_hidden_states = previous_hidden_states.view(-1, hc_mult, H)
|
||||
previous_hidden_states = self.hnorm(previous_hidden_states)
|
||||
|
||||
which lowers to ~6 small kernels (CompareEq, where, Fill, enorm rms_norm,
|
||||
hnorm rms_norm, plus aten elementwise helpers) on the breakable-cudagraph
|
||||
path. Math is preserved: positions==0 → masked row → zero RMS output
|
||||
regardless of weight.
|
||||
|
||||
A single grid (T, hc_mult+1) drives both norms: task 0 is enorm on
|
||||
inputs_embeds[token, :], task k+1 is hnorm on previous_hidden_states[token, k, :].
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_row(
|
||||
x,
|
||||
w_ptr,
|
||||
out_row_ptr,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
):
|
||||
x = x.to(tl.float32)
|
||||
variance = tl.sum(x * x, axis=0) / HIDDEN
|
||||
rrms = tl.rsqrt(variance + eps)
|
||||
w = tl.load(w_ptr + block, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * rrms * w
|
||||
tl.store(out_row_ptr + block, y.to(out_row_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
class FusedMTPInputRMSNormKernel(
|
||||
VllmTritonJitKernel["FusedMTPInputRMSNormKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
hidden: int
|
||||
hc_mult: int
|
||||
block_size: int
|
||||
eps: float
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
inputs_embeds_ptr,
|
||||
positions_ptr,
|
||||
prev_hidden_ptr,
|
||||
enorm_weight_ptr,
|
||||
hnorm_weight_ptr,
|
||||
enorm_out_ptr,
|
||||
hnorm_out_ptr,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
HC_MULT: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
# int64 token index so per-token offsets don't overflow int32 at
|
||||
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
|
||||
if pid_task == 0:
|
||||
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
|
||||
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
|
||||
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
|
||||
pos = tl.load(positions_ptr + token_idx)
|
||||
keep = pos != 0
|
||||
x = tl.load(
|
||||
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
|
||||
)
|
||||
x = tl.where(keep, x, 0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
enorm_weight_ptr,
|
||||
enorm_out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
else:
|
||||
# hnorm path: load prev_hidden[token, slot, :].
|
||||
slot = pid_task - 1
|
||||
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
|
||||
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
hnorm_weight_ptr,
|
||||
hnorm_out_ptr + row_offset,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
hidden: int,
|
||||
hc_mult: int,
|
||||
eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
hidden=hidden,
|
||||
hc_mult=hc_mult,
|
||||
block_size=next_power_of_2(hidden),
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
hidden=int(hf_config.hidden_size),
|
||||
hc_mult=int(hf_config.hc_mult),
|
||||
eps=float(hf_config.rms_norm_eps),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
inputs_embeds=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
previous_hidden_states=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hc_mult, compile_key.hidden),
|
||||
),
|
||||
enorm_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
hnorm_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
_outputs=(
|
||||
TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hc_mult, compile_key.hidden),
|
||||
),
|
||||
),
|
||||
eps=compile_key.eps,
|
||||
hc_mult=compile_key.hc_mult,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
enorm_weight: torch.Tensor,
|
||||
hnorm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_mult: int,
|
||||
*,
|
||||
_outputs: tuple[Any, Any] | None = None,
|
||||
) -> LaunchSpec:
|
||||
num_tokens, hidden = inputs_embeds.shape
|
||||
if _outputs is None:
|
||||
assert inputs_embeds.ndim == 2
|
||||
assert previous_hidden_states.ndim == 3
|
||||
assert previous_hidden_states.shape[1] == hc_mult
|
||||
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
|
||||
"token dim mismatch"
|
||||
)
|
||||
assert (
|
||||
hidden
|
||||
== previous_hidden_states.shape[2]
|
||||
== enorm_weight.shape[0]
|
||||
== hnorm_weight.shape[0]
|
||||
)
|
||||
assert (
|
||||
inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
|
||||
)
|
||||
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
|
||||
_outputs = (
|
||||
torch.empty_like(inputs_embeds),
|
||||
torch.empty_like(previous_hidden_states),
|
||||
)
|
||||
|
||||
enorm_out, hnorm_out = _outputs
|
||||
grid = None if num_tokens == 0 else (num_tokens, hc_mult + 1)
|
||||
return (
|
||||
grid,
|
||||
dict(
|
||||
prev_hidden_ptr=previous_hidden_states,
|
||||
enorm_out_ptr=enorm_out,
|
||||
hnorm_out_ptr=hnorm_out,
|
||||
HIDDEN=hidden,
|
||||
HC_MULT=hc_mult,
|
||||
BLOCK_SIZE=next_power_of_2(hidden),
|
||||
),
|
||||
_outputs,
|
||||
)
|
||||
|
||||
|
||||
class MTPSharedHeadRMSNormKernel(
|
||||
VllmTritonJitKernel["MTPSharedHeadRMSNormKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
hidden: int
|
||||
block_size: int
|
||||
eps: float
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
out_ptr,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
weight_ptr,
|
||||
out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
hidden: int,
|
||||
eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
hidden=hidden,
|
||||
block_size=next_power_of_2(hidden),
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
hidden=int(hf_config.hidden_size),
|
||||
eps=float(hf_config.rms_norm_eps),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
hidden_states=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
_output=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
eps=compile_key.eps,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
*,
|
||||
_output: Any | None = None,
|
||||
) -> LaunchSpec:
|
||||
num_tokens, hidden = hidden_states.shape
|
||||
if _output is None:
|
||||
assert hidden_states.ndim == 2
|
||||
assert hidden_states.is_contiguous()
|
||||
assert weight.is_contiguous()
|
||||
_output = torch.empty_like(hidden_states)
|
||||
|
||||
grid = None if num_tokens == 0 else (num_tokens,)
|
||||
return (
|
||||
grid,
|
||||
dict(
|
||||
x_ptr=hidden_states,
|
||||
out_ptr=_output,
|
||||
HIDDEN=hidden,
|
||||
BLOCK_SIZE=next_power_of_2(hidden),
|
||||
),
|
||||
_output,
|
||||
)
|
||||
|
||||
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL = FusedMTPInputRMSNormKernel()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL = MTPSharedHeadRMSNormKernel()
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
class SavePartialStatesKernel(
|
||||
VllmTritonJitKernel["SavePartialStatesKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
head_size: int
|
||||
triton_block_size: int
|
||||
state_width: int
|
||||
compress_ratio: int
|
||||
kv_stride: int
|
||||
score_stride: int
|
||||
ape_stride: int
|
||||
state_cache_stride0: int
|
||||
state_cache_stride1: int
|
||||
block_size: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
kv_ptr,
|
||||
kv_stride,
|
||||
score_ptr,
|
||||
score_stride,
|
||||
ape_ptr,
|
||||
ape_stride,
|
||||
positions_ptr,
|
||||
state_cache_ptr,
|
||||
state_cache_stride0,
|
||||
state_cache_stride1,
|
||||
slot_mapping_ptr,
|
||||
block_size,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
slot_id = tl.load(slot_mapping_ptr + token_idx)
|
||||
|
||||
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
|
||||
# by vLLM). During CUDA graph replay the batch may contain padding
|
||||
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
|
||||
# illegal memory access.
|
||||
if slot_id < 0:
|
||||
return
|
||||
|
||||
block_idx = slot_id // block_size
|
||||
pos_in_block = slot_id % block_size
|
||||
base_ptr = (
|
||||
state_cache_ptr
|
||||
+ block_idx * state_cache_stride0
|
||||
+ pos_in_block * state_cache_stride1
|
||||
)
|
||||
|
||||
block = tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = block < HEAD_SIZE
|
||||
|
||||
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
|
||||
tl.store(base_ptr + block, kv, mask=mask)
|
||||
|
||||
# Fused: score += ape[position % compress_ratio]
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
ape_row = position % COMPRESS_RATIO
|
||||
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
|
||||
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
|
||||
tl.store(
|
||||
base_ptr + STATE_WIDTH + block,
|
||||
score + ape,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
head_size: int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
head_size=head_size,
|
||||
triton_block_size=next_power_of_2(head_size),
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
head_dim: int,
|
||||
compress_ratio: int,
|
||||
) -> list[CompileKey]:
|
||||
if head_dim <= 0 or compress_ratio not in (4, 128):
|
||||
return []
|
||||
|
||||
coefficient = 2 if compress_ratio == 4 else 1
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_size=coefficient * head_dim,
|
||||
state_width=coefficient * head_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
kv_stride=2 * coefficient * head_dim,
|
||||
score_stride=2 * coefficient * head_dim,
|
||||
ape_stride=coefficient * head_dim,
|
||||
state_cache_stride0=8 * coefficient * head_dim,
|
||||
state_cache_stride1=2 * coefficient * head_dim,
|
||||
block_size=4 if compress_ratio == 4 else 8,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
kv=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.head_size),
|
||||
strides=(compile_key.kv_stride, 1),
|
||||
),
|
||||
score=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1),
|
||||
strides=(compile_key.score_stride, 1),
|
||||
),
|
||||
ape=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1),
|
||||
strides=(compile_key.ape_stride, 1),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
state_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1, compile_key.state_width),
|
||||
strides=(
|
||||
compile_key.state_cache_stride0,
|
||||
compile_key.state_cache_stride1,
|
||||
1,
|
||||
),
|
||||
),
|
||||
slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
block_size=compile_key.block_size,
|
||||
state_width=compile_key.state_width,
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
kv: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
state_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
block_size: int,
|
||||
state_width: int,
|
||||
compress_ratio: int,
|
||||
pdl_kwargs: dict | None = None,
|
||||
) -> LaunchSpec:
|
||||
"""Write packed [kv, score+ape] partial states into the compressor cache.
|
||||
|
||||
One program per token; pads (slot_id == -1) are skipped.
|
||||
"""
|
||||
num_actual = slot_mapping.shape[0]
|
||||
head_size = kv.shape[-1]
|
||||
return (num_actual,), dict(
|
||||
kv_stride=kv.stride(0),
|
||||
score_stride=score.stride(0),
|
||||
ape_stride=ape.stride(0),
|
||||
state_cache_stride0=state_cache.stride(0),
|
||||
state_cache_stride1=state_cache.stride(1),
|
||||
block_size=block_size,
|
||||
HEAD_SIZE=head_size,
|
||||
TRITON_BLOCK_SIZE=next_power_of_2(head_size),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
**(pdl_kwargs or {}),
|
||||
)
|
||||
|
||||
|
||||
_SAVE_PARTIAL_STATES_KERNEL = SavePartialStatesKernel()
|
||||
@@ -1,52 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepseekV4 rotary embedding initialization."""
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.base import RotaryEmbedding
|
||||
|
||||
|
||||
def build_deepseek_v4_rope(
|
||||
config,
|
||||
*,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
max_position_embeddings: int,
|
||||
compress_ratio: int,
|
||||
) -> RotaryEmbedding:
|
||||
rope_parameters = config.rope_parameters
|
||||
# Newer checkpoints nest per-layer-type rope dicts ({"main", "compress"});
|
||||
# older ones ship a single flat dict shared by all layer types.
|
||||
if isinstance(rope_parameters.get("main"), dict) and isinstance(
|
||||
rope_parameters.get("compress"), dict
|
||||
):
|
||||
key = "compress" if compress_ratio > 1 else "main"
|
||||
rope_parameters = dict(rope_parameters[key])
|
||||
else:
|
||||
rope_parameters = dict(rope_parameters)
|
||||
|
||||
rope_parameters["rope_theta"] = (
|
||||
config.compress_rope_theta if compress_ratio > 1 else config.rope_theta
|
||||
)
|
||||
if compress_ratio > 1 and rope_parameters["rope_type"] != "default":
|
||||
# YaRN applies only to compressor (CSA/HCA) layers.
|
||||
rope_parameters["rope_type"] = (
|
||||
"deepseek_llama_scaling"
|
||||
if rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
else:
|
||||
# Sliding-window layers use plain RoPE (theta=rope_theta, no YaRN).
|
||||
rope_parameters["rope_type"] = "deepseek_yarn"
|
||||
rope_parameters["factor"] = 1.0
|
||||
rope_parameters["original_max_position_embeddings"] = max_position_embeddings
|
||||
rope_parameters["mscale"] = 0 # Disable mscale
|
||||
rope_parameters["mscale_all_dim"] = 0 # Disable mscale
|
||||
rope_parameters["is_deepseek_v4"] = True
|
||||
rope_parameters["rope_dim"] = rope_head_dim
|
||||
return get_rope(
|
||||
head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
@@ -1,476 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 vision tower (ViT + aligner) with TP-sharded linears.
|
||||
|
||||
Ported from the official reference implementation
|
||||
(deepseek-ai/DeepSeek-V4-Flash-Vision-Exp). Weight names match the HF
|
||||
checkpoint so no renaming is needed at load time. Attention and MLP weights
|
||||
are tensor-parallel sharded (replicated when the vision head count is not
|
||||
divisible by TP size, or under ``--mm-encoder-tp-mode data``); the patch
|
||||
embed and norms are replicated, so the residual stream is full-width on
|
||||
every rank.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_load_balance_assignment,
|
||||
is_vit_use_data_parallel,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
def _compute_vision_cos_sin(
|
||||
n_h: int, n_w: int, dim: int, theta: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
|
||||
hpos = torch.arange(n_h).unsqueeze(1).expand(n_h, n_w)
|
||||
wpos = torch.arange(n_w).unsqueeze(0).expand(n_h, n_w)
|
||||
freqs = torch.stack([hpos, wpos], dim=-1).reshape(-1, 2, 1).float()
|
||||
freqs = (freqs * inv_freq).flatten(1)
|
||||
return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)
|
||||
|
||||
|
||||
@lru_cache(8)
|
||||
def get_vision_cos_sin(
|
||||
n_h: int, n_w: int, dim: int, theta: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return _compute_vision_cos_sin(n_h, n_w, dim, theta)
|
||||
|
||||
|
||||
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
||||
dtype = x.dtype
|
||||
x1, x2 = x.float().chunk(2, dim=-1)
|
||||
return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(dtype)
|
||||
|
||||
|
||||
class DeepseekV4RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
dtype = x.dtype
|
||||
x = x.float()
|
||||
x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)
|
||||
return (self.weight * x).to(dtype)
|
||||
|
||||
|
||||
class DeepseekV4PatchEmbed(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
# Replicated: the residual stream is full-width on every rank.
|
||||
self.proj = ReplicatedLinear(
|
||||
3 * config.vision_patch_size**2,
|
||||
config.vision_dim,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
out, _ = self.proj(x.flatten(1))
|
||||
return out
|
||||
|
||||
|
||||
class DeepseekV4VisionAttention(nn.Module):
|
||||
def __init__(self, config, prefix: str = ""):
|
||||
super().__init__()
|
||||
# Convention from minimax_m3/siglip: compute the TP/DP choice locally
|
||||
# instead of threading a flag through every constructor.
|
||||
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
|
||||
self.tp_size = (
|
||||
1 if use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.n_heads = config.vision_n_heads // self.tp_size
|
||||
self.head_dim = config.vision_dim // config.vision_n_heads
|
||||
self.hidden_size = config.vision_dim
|
||||
self.wqkv = QKVParallelLinear(
|
||||
config.vision_dim,
|
||||
self.head_dim,
|
||||
config.vision_n_heads,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.wqkv",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.wo = RowParallelLinear(
|
||||
config.vision_dim,
|
||||
config.vision_dim,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.wo",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.attn = MMEncoderAttention(
|
||||
num_heads=self.n_heads,
|
||||
head_size=self.head_dim,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
n = x.size(0)
|
||||
qkv, _ = self.wqkv(x)
|
||||
q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in qkv.chunk(3, -1))
|
||||
q = apply_rotary(q, cos, sin).unsqueeze(0) # (b=1, n, h, d)
|
||||
k = apply_rotary(k, cos, sin).unsqueeze(0)
|
||||
# Dense (cu_seqlens=None): one image per call. Varlen (cu_seqlens
|
||||
# set): packed multi-image batch for encoder CUDA graph replay.
|
||||
o = self.attn(
|
||||
q, k, v.unsqueeze(0), cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
|
||||
)
|
||||
out, _ = self.wo(o.reshape(n, -1))
|
||||
return out
|
||||
|
||||
|
||||
class DeepseekV4VisionMLP(nn.Module):
|
||||
def __init__(self, config, prefix: str = ""):
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
|
||||
self.w1 = MergedColumnParallelLinear(
|
||||
config.vision_dim,
|
||||
[config.vision_inter_dim] * 2,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.w1",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.w2 = RowParallelLinear(
|
||||
config.vision_inter_dim,
|
||||
config.vision_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.w2",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.w1(x)
|
||||
out, _ = self.w2(self.act_fn(gate_up))
|
||||
return out
|
||||
|
||||
|
||||
class DeepseekV4VisionBlock(nn.Module):
|
||||
def __init__(self, config, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.norm1 = DeepseekV4RMSNorm(config.vision_dim)
|
||||
self.attn = DeepseekV4VisionAttention(config, prefix=f"{prefix}.attn")
|
||||
self.norm2 = DeepseekV4RMSNorm(config.vision_dim)
|
||||
self.mlp = DeepseekV4VisionMLP(config, prefix=f"{prefix}.mlp")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = x + self.attn(self.norm1(x), cos, sin, cu_seqlens, max_seqlen)
|
||||
return x + self.mlp(self.norm2(x))
|
||||
|
||||
|
||||
class DeepseekV4ViT(nn.Module):
|
||||
"""DeepSeek-V4 ViT: full bidirectional attention per image, 2D RoPE."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.rope_dim = config.vision_dim // config.vision_n_heads // 2
|
||||
self.rope_theta = config.vision_rope_theta
|
||||
self.patch_embed = DeepseekV4PatchEmbed(config)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4VisionBlock(config, prefix=f"blocks.{i}")
|
||||
for i in range(config.vision_n_layers)
|
||||
]
|
||||
)
|
||||
self.norm = DeepseekV4RMSNorm(config.vision_dim)
|
||||
|
||||
def forward(
|
||||
self, patches: torch.Tensor, n_vit_h: int, n_vit_w: int
|
||||
) -> torch.Tensor:
|
||||
x = self.patch_embed(patches)
|
||||
cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta)
|
||||
cos = async_tensor_h2d(cos, x.device)
|
||||
sin = async_tensor_h2d(sin, x.device)
|
||||
for block in self.blocks:
|
||||
x = block(x, cos, sin)
|
||||
return self.norm(x)
|
||||
|
||||
def forward_packed(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Varlen path for encoder CUDA graphs: multiple images packed along
|
||||
rows, per-image attention via ``cu_seqlens``, RoPE tables precomputed
|
||||
on device. Pure-tensor (no host reads, no H2D), safe to capture."""
|
||||
x = self.patch_embed(patches)
|
||||
for block in self.blocks:
|
||||
x = block(x, cos, sin, cu_seqlens, max_seqlen)
|
||||
return self.norm(x)
|
||||
|
||||
|
||||
def build_packed_vit_metadata(
|
||||
grids: list[list[int]] | list[tuple[int, int]],
|
||||
*,
|
||||
rope_dim: int,
|
||||
rope_theta: float,
|
||||
device: torch.device,
|
||||
max_seqlen_override: int | None = None,
|
||||
cached: bool = True,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Precompute packed-batch ViT metadata for ``DeepseekV4ViT.forward_packed``.
|
||||
|
||||
Args:
|
||||
grids: ``[n_vit_h, n_vit_w]`` per image, in packing order.
|
||||
rope_dim: RoPE embedding dimension of the ViT.
|
||||
rope_theta: RoPE base frequency of the ViT.
|
||||
device: Device for the returned tensors (except ``max_seqlen``,
|
||||
which stays on the host).
|
||||
max_seqlen_override: Worst-case value baked in at CUDA graph capture
|
||||
(the attention wrapper reads ``max_seqlen`` on the host, so the
|
||||
capture-time value becomes a graph constant).
|
||||
cached: Use the shared ``lru_cache`` for RoPE tables. Capture-time
|
||||
dummy grids pass ``False`` to avoid evicting real entries.
|
||||
|
||||
"""
|
||||
cos_sin = get_vision_cos_sin if cached else _compute_vision_cos_sin
|
||||
cos_list: list[torch.Tensor] = []
|
||||
sin_list: list[torch.Tensor] = []
|
||||
cu_seqlens = [0]
|
||||
max_seqlen = 0
|
||||
for n_h, n_w in grids:
|
||||
cos, sin = cos_sin(n_h, n_w, rope_dim, rope_theta)
|
||||
cos_list.append(cos)
|
||||
sin_list.append(sin)
|
||||
n = n_h * n_w
|
||||
cu_seqlens.append(cu_seqlens[-1] + n)
|
||||
max_seqlen = max(max_seqlen, n)
|
||||
if cos_list:
|
||||
cos = async_tensor_h2d(torch.cat(cos_list), device)
|
||||
sin = async_tensor_h2d(torch.cat(sin_list), device)
|
||||
else:
|
||||
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
if max_seqlen_override is not None:
|
||||
max_seqlen = max_seqlen_override
|
||||
return {
|
||||
"vit_cos": cos,
|
||||
"vit_sin": sin,
|
||||
"cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32),
|
||||
# Read on the host by the attention wrapper; keep on CPU.
|
||||
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
|
||||
}
|
||||
|
||||
|
||||
def build_packed_merge_metadata(
|
||||
grids: list[list[int]] | list[tuple[int, int]],
|
||||
downsample_ratio: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Gather indices/mask for ``DeepseekV4Aligner.forward_packed``.
|
||||
|
||||
For each aligner output row, the ``r x r`` source patch rows in the
|
||||
packed ViT output; positions past the image edge (the eager path pads
|
||||
them with zeros) get mask 0 and a clamped in-range index.
|
||||
"""
|
||||
r = downsample_ratio
|
||||
idx_list: list[torch.Tensor] = []
|
||||
mask_list: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for n_h, n_w in grids:
|
||||
rows = -(-n_h // r)
|
||||
cols = -(-n_w // r)
|
||||
bi = torch.arange(rows).repeat_interleave(cols)
|
||||
bj = torch.arange(cols).repeat(rows)
|
||||
di = torch.arange(r).repeat_interleave(r)
|
||||
dj = torch.arange(r).repeat(r)
|
||||
hi = bi[:, None] * r + di[None, :]
|
||||
wj = bj[:, None] * r + dj[None, :]
|
||||
valid = (hi < n_h) & (wj < n_w)
|
||||
idx = offset + hi.clamp(max=n_h - 1) * n_w + wj.clamp(max=n_w - 1)
|
||||
idx_list.append(idx)
|
||||
mask_list.append(valid)
|
||||
offset += n_h * n_w
|
||||
if idx_list:
|
||||
merge_idx = async_tensor_h2d(torch.cat(idx_list), device)
|
||||
merge_mask = async_tensor_h2d(
|
||||
torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype
|
||||
)
|
||||
else:
|
||||
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
|
||||
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
|
||||
return {"merge_idx": merge_idx, "merge_mask": merge_mask}
|
||||
|
||||
|
||||
class DeepseekV4Aligner(nn.Module):
|
||||
"""Spatial merge (downsample_ratio x downsample_ratio) + MLP projector."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
|
||||
self.downsample_ratio = config.vision_downsample_ratio
|
||||
self.out_dim = config.hidden_size
|
||||
in_dim = config.vision_dim * self.downsample_ratio**2
|
||||
self.w1 = ColumnParallelLinear(
|
||||
in_dim,
|
||||
config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.w2 = RowParallelLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, n_vit_h: int, n_vit_w: int) -> torch.Tensor:
|
||||
r = self.downsample_ratio
|
||||
x = x.view(n_vit_h, n_vit_w, -1).permute(2, 0, 1)
|
||||
x = F.pad(x, (0, -n_vit_w % r, 0, -n_vit_h % r))
|
||||
x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).transpose(0, 1)
|
||||
hidden, _ = self.w1(x)
|
||||
out, _ = self.w2(F.gelu(hidden))
|
||||
return out
|
||||
|
||||
def forward_packed(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
merge_idx: torch.Tensor,
|
||||
merge_mask: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Packed multi-image path for encoder CUDA graphs.
|
||||
|
||||
``merge_idx``/``merge_mask`` come from ``build_packed_merge_metadata``;
|
||||
gather+mask reproduces the eager ``F.pad`` + ``F.unfold`` merge
|
||||
exactly (unfold rows are channel-major, so the gathered patch rows
|
||||
are transposed before flattening).
|
||||
"""
|
||||
r2 = self.downsample_ratio**2
|
||||
m = merge_idx.shape[0]
|
||||
gathered = (x[merge_idx] * merge_mask).view(m, r2, -1)
|
||||
gathered = gathered.permute(0, 2, 1).reshape(m, -1)
|
||||
hidden, _ = self.w1(gathered)
|
||||
out, _ = self.w2(F.gelu(hidden))
|
||||
return out
|
||||
|
||||
|
||||
def run_dp_sharded_vision_tower(
|
||||
vision_model: DeepseekV4ViT,
|
||||
aligner: DeepseekV4Aligner,
|
||||
patches: torch.Tensor,
|
||||
vit_grid: list[list[int]],
|
||||
) -> list[torch.Tensor]:
|
||||
"""Run the ViT + aligner with images sharded across TP ranks.
|
||||
|
||||
Every rank holds the full tower weights (``--mm-encoder-tp-mode data``)
|
||||
and receives the full ``patches`` batch. Images are assigned to ranks by
|
||||
patch count (greedy load balancing), each rank encodes only its share,
|
||||
and per-image embeddings are exchanged with one padded all-gather and
|
||||
returned in the original image order.
|
||||
|
||||
Args:
|
||||
vision_model: The (weight-replicated) ViT tower.
|
||||
aligner: The (weight-replicated) spatial-merge projector.
|
||||
patches: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` patches of all images.
|
||||
vit_grid: ``[n_vit_h, n_vit_w]`` per image.
|
||||
|
||||
Returns:
|
||||
One ``(n_aligner_rows, hidden_size)`` embedding tensor per image.
|
||||
|
||||
"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
sizes = [h * w for h, w in vit_grid]
|
||||
cum_patches = [0, *itertools.accumulate(sizes)]
|
||||
image_to_tp_rank, gpu_sample_counts, _ = get_load_balance_assignment(sizes, tp_size)
|
||||
cum_sample_counts = [0, *itertools.accumulate(gpu_sample_counts)]
|
||||
|
||||
# Rows the aligner emits per image: each grid dim is padded up to a
|
||||
# multiple of the merge ratio before r x r patches fold into one row.
|
||||
r = aligner.downsample_ratio
|
||||
rows_per_image = [-(-h // r) * -(-w // r) for h, w in vit_grid]
|
||||
|
||||
def rank_image_idxs(g: int) -> list[int]:
|
||||
return image_to_tp_rank[cum_sample_counts[g] : cum_sample_counts[g + 1]]
|
||||
|
||||
# All-gather needs one shape on every rank; pad each rank's packed
|
||||
# output to the largest per-rank row count (computable locally).
|
||||
max_rows = max(
|
||||
sum(rows_per_image[i] for i in rank_image_idxs(g)) for g in range(tp_size)
|
||||
)
|
||||
|
||||
local_embeds = [
|
||||
aligner(
|
||||
vision_model(
|
||||
patches[cum_patches[i] : cum_patches[i + 1]],
|
||||
vit_grid[i][0],
|
||||
vit_grid[i][1],
|
||||
),
|
||||
vit_grid[i][0],
|
||||
vit_grid[i][1],
|
||||
)
|
||||
for i in rank_image_idxs(tp_rank)
|
||||
]
|
||||
if local_embeds:
|
||||
embeds_local = torch.cat(local_embeds, dim=0)
|
||||
else:
|
||||
embeds_local = patches.new_zeros((0, aligner.out_dim))
|
||||
if embeds_local.shape[0] < max_rows:
|
||||
embeds_local = torch.cat(
|
||||
[
|
||||
embeds_local,
|
||||
embeds_local.new_zeros(
|
||||
(max_rows - embeds_local.shape[0], embeds_local.shape[1])
|
||||
),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
gathered = tensor_model_parallel_all_gather(embeds_local.contiguous(), dim=0)
|
||||
|
||||
out: list[torch.Tensor] = [None] * len(vit_grid) # type: ignore[list-item]
|
||||
for g in range(tp_size):
|
||||
offset = g * max_rows
|
||||
for i in rank_image_idxs(g):
|
||||
out[i] = gathered[offset : offset + rows_per_image[i]]
|
||||
offset += rows_per_image[i]
|
||||
return out
|
||||
@@ -1,393 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 vision variant (e.g. DeepSeek-V4-Flash-Vision-Exp).
|
||||
|
||||
Thin multimodal wrapper around the text-only ``DeepseekV4ForCausalLM``:
|
||||
|
||||
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
|
||||
sentinel positions; four learned vectors (``image_start`` / ``image_pad`` /
|
||||
``image_newline`` / ``image_end``) fill the remaining sentinel positions.
|
||||
- Image placeholders (``<|deepseek_image|>``) are expanded by the processor
|
||||
in ``common/mm_preprocess.py`` into sentinel blocks borrowing reserved
|
||||
in-vocab tokens ``<|place_holder_mm_span_0431|>``..``_0435|>``
|
||||
(see ``common/vision.py`` for the tower itself).
|
||||
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
|
||||
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
|
||||
the model so the MoE router can apply ``bias_vl`` to image tokens
|
||||
(``requires_raw_input_tokens``).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
|
||||
from .mm_preprocess import (
|
||||
COMPRESS_PAD_TO,
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
DeepseekV4VLProcessingInfo,
|
||||
image_sentinel_mask,
|
||||
)
|
||||
from .vision import DeepseekV4Aligner, DeepseekV4ViT
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(
|
||||
text_mapper: WeightsMapper, image_enabled: bool
|
||||
) -> WeightsMapper:
|
||||
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
|
||||
orig_to_new_prefix = {
|
||||
src: None if dst is None else f"language_model.{dst}"
|
||||
for src, dst in text_mapper.orig_to_new_prefix.items()
|
||||
}
|
||||
if not image_enabled:
|
||||
orig_to_new_prefix.update({"vision.": None, "aligner.": None, "image_": None})
|
||||
return WeightsMapper(
|
||||
orig_to_new_renaming=text_mapper.orig_to_new_renaming,
|
||||
orig_to_new_prefix=orig_to_new_prefix,
|
||||
orig_to_new_regex=text_mapper.orig_to_new_regex,
|
||||
orig_to_new_stacked=text_mapper.orig_to_new_stacked,
|
||||
orig_to_new_suffix={
|
||||
**text_mapper.orig_to_new_suffix,
|
||||
"head.weight": "language_model.lm_head.weight",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
**text_mapper.orig_to_new_substr,
|
||||
# Draft models load the checkpoint's MTP weights separately from
|
||||
# the target model.
|
||||
"mtp.": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def stream_language_model_first(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield ``language_model.*`` weights as they arrive, then the rest.
|
||||
|
||||
Keeps the language model's weights in one contiguous group without
|
||||
materializing the stream. Deferred weights are cloned so they do not keep
|
||||
the loader's per-shard storage alive until the end of the load.
|
||||
"""
|
||||
rest = []
|
||||
for name, tensor in weights:
|
||||
if name.startswith("language_model."):
|
||||
yield name, tensor
|
||||
else:
|
||||
rest.append((name, tensor.clone()))
|
||||
yield from rest
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
||||
)
|
||||
class DeepseekV4ForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3, SupportsLoRA
|
||||
):
|
||||
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
|
||||
|
||||
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
|
||||
delegates through ``language_model`` via the protocol defaults.
|
||||
"""
|
||||
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
"fused_wkv_wgate": ["wkv", "wgate"],
|
||||
# for visual encoder
|
||||
"wqkv": ["wqkv"],
|
||||
"w1": ["w1"],
|
||||
}
|
||||
|
||||
# The MoE router needs raw token ids to detect image sentinel tokens
|
||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
supports_tower_connector_lora = True
|
||||
# Image spans widen the sparse-SWA window in-kernel and the placeholder
|
||||
# starts with a compressor-alignment pad
|
||||
mm_prefix_clamp_sliding_window = True
|
||||
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
return IMAGE_PLACEHOLDER
|
||||
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||
|
||||
def __init__(self, *, vllm_config, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config = model_config.hf_config
|
||||
self.config = config
|
||||
self.multimodal_config = model_config.multimodal_config
|
||||
assert self.multimodal_config is not None
|
||||
|
||||
image_enabled = (
|
||||
config.vision_n_layers > 0
|
||||
and self.multimodal_config.get_limit_per_prompt("image") > 0
|
||||
)
|
||||
with self._mark_tower_model(vllm_config, {"image"}):
|
||||
self.vision: DeepseekV4ViT | None = None
|
||||
self.aligner: DeepseekV4Aligner | None = None
|
||||
self.image_start: nn.Parameter | None = None
|
||||
self.image_end: nn.Parameter | None = None
|
||||
self.image_newline: nn.Parameter | None = None
|
||||
self.image_pad: nn.Parameter | None = None
|
||||
if image_enabled:
|
||||
self.vision = DeepseekV4ViT(config)
|
||||
self.aligner = DeepseekV4Aligner(config)
|
||||
for name in (
|
||||
"image_start",
|
||||
"image_end",
|
||||
"image_newline",
|
||||
"image_pad",
|
||||
):
|
||||
setattr(
|
||||
self,
|
||||
name,
|
||||
nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
),
|
||||
)
|
||||
self.vision.to(dtype=model_config.dtype)
|
||||
self.aligner.to(dtype=model_config.dtype)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
# The arch convertor routes any config with a vision tower to
|
||||
# this wrapper class; mark the copy handed to the text backbone
|
||||
# so it resolves to DeepseekV4ForCausalLM instead of recursing
|
||||
# (with_hf_config deepcopies the config, the marker survives).
|
||||
config._dsv4_vl_inner = True # type: ignore[attr-defined]
|
||||
try:
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
finally:
|
||||
del config._dsv4_vl_inner # type: ignore[attr-defined]
|
||||
# The outer mapper (see load_weights) fully resolves HF names into
|
||||
# this wrapper's namespace before AutoWeightsLoader strips the
|
||||
# "language_model." prefix and delegates to the child's load_weights,
|
||||
# so the child's own mapper must be a no-op. Its suffix rules are not
|
||||
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
|
||||
# re-fire "head.weight" -> "lm_head.weight").
|
||||
text_mapper = self.language_model.hf_to_vllm_mapper
|
||||
self.language_model.hf_to_vllm_mapper = WeightsMapper()
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
|
||||
text_mapper, image_enabled
|
||||
)
|
||||
self._weights_finalized = False
|
||||
|
||||
def _parse_and_validate_image_input(self, **kwargs: object) -> dict | None:
|
||||
patches = kwargs.pop("patches", None)
|
||||
if patches is None:
|
||||
return None
|
||||
vit_grid = kwargs.pop("vit_grid", None)
|
||||
llm_grid = kwargs.pop("llm_grid", None)
|
||||
perm = kwargs.pop("perm", None)
|
||||
assert vit_grid is not None and llm_grid is not None and perm is not None
|
||||
return {
|
||||
"patches": patches,
|
||||
"vit_grid": vit_grid,
|
||||
"llm_grid": llm_grid,
|
||||
"perm": perm,
|
||||
}
|
||||
|
||||
def _encode_image(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
n_vit_h: int,
|
||||
n_vit_w: int,
|
||||
perm: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
image_embeds = self.aligner(
|
||||
self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w
|
||||
)
|
||||
# Reorder into the N-layout block order used in the prompt.
|
||||
return image_embeds[perm.to(image_embeds.device)]
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
vit_grid: torch.Tensor,
|
||||
llm_grid: torch.Tensor,
|
||||
perm: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
patches = patches.to(self.aligner.w1.weight.dtype)
|
||||
|
||||
embeds: list[torch.Tensor] = []
|
||||
vit_offset = 0
|
||||
llm_offset = 0
|
||||
for (n_vit_h, n_vit_w), (n_llm_h, n_llm_w) in zip(
|
||||
vit_grid.tolist(), llm_grid.tolist(), strict=True
|
||||
):
|
||||
n_vit = n_vit_h * n_vit_w
|
||||
n_llm = n_llm_h * n_llm_w
|
||||
embeds.append(
|
||||
self._encode_image(
|
||||
patches[vit_offset : vit_offset + n_vit],
|
||||
n_vit_h,
|
||||
n_vit_w,
|
||||
perm[llm_offset : llm_offset + n_llm],
|
||||
)
|
||||
)
|
||||
vit_offset += n_vit
|
||||
llm_offset += n_llm
|
||||
return tuple(embeds)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None or self.vision is None:
|
||||
return []
|
||||
return self._process_image_input(
|
||||
image_input["patches"],
|
||||
image_input["vit_grid"],
|
||||
image_input["llm_grid"],
|
||||
image_input["perm"],
|
||||
)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
# All ids are in-vocab here: image-block sentinels are borrowed
|
||||
# reserved tokens (their embedding rows are always overwritten below).
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
if self.image_start is not None:
|
||||
# Branch-free sentinel overwrite: safe inside compiled/captured
|
||||
# regions (no data-dependent control flow).
|
||||
sentinel_mask = image_sentinel_mask(input_ids)
|
||||
if is_multimodal is not None:
|
||||
# IMAGE positions get vision embeddings via the merge below.
|
||||
sentinel_mask = sentinel_mask & ~is_multimodal.to(input_ids.device)
|
||||
table = torch.stack(
|
||||
[
|
||||
self.image_start,
|
||||
self.image_pad,
|
||||
self.image_pad,
|
||||
self.image_newline,
|
||||
self.image_end,
|
||||
]
|
||||
).to(inputs_embeds.dtype)
|
||||
idx = (input_ids - IMAGE_SENTINEL_BASE_ID).clamp(0, 4)
|
||||
inputs_embeds = torch.where(
|
||||
sentinel_mask.unsqueeze(-1), table[idx], inputs_embeds
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return self.language_model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.language_model.get_expert_mapping()
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
child_finalizes = getattr(
|
||||
self.language_model, "finalizes_weights_during_load", True
|
||||
)
|
||||
mapped = self.hf_to_vllm_mapper.apply(weights)
|
||||
if child_finalizes:
|
||||
# A child which finalizes inside load_weights must see all of its
|
||||
# weights in one contiguous delegation from AutoWeightsLoader.
|
||||
mapped = stream_language_model_first(mapped)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
self._weights_finalized = child_finalizes
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Backbones such as the ROCm implementation require this to run only
|
||||
# after the loader's generic per-layer quantization finalization.
|
||||
if getattr(self, "_weights_finalized", False):
|
||||
return
|
||||
self.language_model.process_weights_after_loading()
|
||||
self._weights_finalized = True
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""Get the module prefixes in the multimodal model."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="aligner",
|
||||
tower_model="vision.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
if modality != "image":
|
||||
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||
|
||||
patches = mm_kwargs.get("patches") if mm_kwargs else None
|
||||
if patches is not None and isinstance(patches.data, torch.Tensor):
|
||||
tower_tokens = patches.data.shape[0]
|
||||
else:
|
||||
tower_tokens = num_mm_embeds * self.config.vision_downsample_ratio**2
|
||||
|
||||
return tower_tokens, num_mm_embeds
|
||||
@@ -1,499 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
compress_norm_rope_store_triton,
|
||||
compress_norm_rope_store_two_stage_triton,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
from vllm.models.deepseek_v4.common.ops.save_partial_states import (
|
||||
_SAVE_PARTIAL_STATES_KERNEL,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
|
||||
|
||||
def _prefer_two_stage_compressor() -> bool:
|
||||
# Platforms that favor the triton variant of two-stage compressor split.
|
||||
# Currently only tested on ROCm
|
||||
return current_platform.is_rocm()
|
||||
|
||||
|
||||
def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:
|
||||
seq_lens_cpu = metadata.seq_lens_cpu_upper_bound
|
||||
if seq_lens_cpu is None:
|
||||
return None
|
||||
|
||||
query_lens = metadata.query_start_loc_cpu[1:] - metadata.query_start_loc_cpu[:-1]
|
||||
starts = seq_lens_cpu - query_lens
|
||||
starts_list = starts.tolist()
|
||||
query_start_loc = metadata.query_start_loc_cpu.tolist()
|
||||
return any(
|
||||
start % 128 + query_start_loc[i + 1] - query_start_loc[i] >= 128
|
||||
for i, start in enumerate(starts_list)
|
||||
)
|
||||
|
||||
|
||||
class CompressorBackend(AttentionBackend):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CompressorBackend"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [512, 1024]
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
|
||||
return CompressorMetadataBuilder
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompressorMetadata:
|
||||
block_table: torch.Tensor
|
||||
slot_mapping: torch.Tensor
|
||||
block_size: int
|
||||
|
||||
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
|
||||
num_decode_tokens: int | None = None
|
||||
c128_boundary: bool | None = None
|
||||
|
||||
|
||||
class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert isinstance(self.kv_cache_spec, SlidingWindowMLASpec | MLAAttentionSpec)
|
||||
mla_spec = cast(SlidingWindowMLASpec | MLAAttentionSpec, self.kv_cache_spec)
|
||||
self.block_size = mla_spec.block_size
|
||||
|
||||
self.token_to_req_indices = torch.zeros(
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> CompressorMetadata:
|
||||
token_to_req_indices = common_attn_metadata.token_to_req_indices(
|
||||
self.token_to_req_indices
|
||||
)
|
||||
num_decode_tokens = None
|
||||
if _prefer_two_stage_compressor():
|
||||
_, _, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||
common_attn_metadata, decode_threshold=1
|
||||
)
|
||||
async_spec_decode = (
|
||||
self.vllm_config.scheduler_config.async_scheduling
|
||||
and self.vllm_config.speculative_config is not None
|
||||
)
|
||||
return CompressorMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor.clamp_(min=0),
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
block_size=self.block_size,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
c128_boundary=(
|
||||
_get_c128_boundary(common_attn_metadata)
|
||||
if self.block_size == 8 and not async_spec_decode
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
def __init__(
|
||||
self,
|
||||
state_dim: int,
|
||||
dtype: torch.dtype,
|
||||
compress_ratio: int,
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
self.state_dim = state_dim
|
||||
self.dtype = dtype
|
||||
self.prefix = prefix
|
||||
self.kv_cache = torch.tensor([])
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
assert self.dtype == torch.float32
|
||||
assert compress_ratio in [4, 128]
|
||||
coff = 1 + (compress_ratio == 4)
|
||||
self.sliding_window = coff * compress_ratio
|
||||
# Block size is constrained by tensor sharing between compressor states
|
||||
# and KV blocks. Since compressor states share the same physical tensor
|
||||
# as KV blocks, they must use the same page size.
|
||||
# The KV block shape [256//4, head_dim] = [64, 584] determines:
|
||||
# - C4 compressor block shape [4, 2*512*2*4] -> block_size = 4
|
||||
# - C128 compressor block shape [8, 512*2*4] -> block_size = 8
|
||||
# TODO(yifan): make block size automatically determined and configurable.
|
||||
if compress_ratio == 4:
|
||||
self.block_size = 4
|
||||
elif compress_ratio == 128:
|
||||
self.block_size = 8
|
||||
else:
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
|
||||
# full-cache rows share state pages with contiguous KV pages, so padding
|
||||
# would break page matching.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576 if uses_fp8_ds_mla_layout else 512,
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return CompressorBackend
|
||||
|
||||
|
||||
class DeepseekCompressor(nn.Module):
|
||||
"""DeepSeek V4 KV/score compressor.
|
||||
|
||||
Owns the linear / norm / state-cache / ape state and the shared forward
|
||||
prologue (kv/score split, save_partial_states launch). The
|
||||
compress → norm → RoPE → store step is dispatched to a triton kernel
|
||||
(``compress_norm_rope_store_triton``) by default, except for the NVIDIA
|
||||
head_dim=512 path which uses the CuTeDSL compressor kernels for better
|
||||
performance.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
compress_ratio: int,
|
||||
hidden_size: int,
|
||||
head_dim: int,
|
||||
rotate: bool = False,
|
||||
prefix: str = "",
|
||||
k_cache_prefix="",
|
||||
use_fp4_cache: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.compress_ratio = compress_ratio
|
||||
self.hidden_size = hidden_size
|
||||
self.head_dim = head_dim
|
||||
self.rotate = rotate
|
||||
self.prefix = prefix
|
||||
self.k_cache_prefix = k_cache_prefix
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.rope_head_dim = config.qk_rope_head_dim
|
||||
self.nope_head_dim = self.head_dim - self.rope_head_dim
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.device = current_platform.device_type
|
||||
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
|
||||
self.overlap = compress_ratio == 4
|
||||
self.coff = 1 + self.overlap
|
||||
|
||||
# The head=512 cr>=128 no-overlap deep gather uses the two-stage
|
||||
# compressor, which needs an fp32 scratch [max_batched, 512] for
|
||||
# the intermediate compressed_kv.
|
||||
# Currently only tested on ROCm
|
||||
self._use_two_stage_fused_compressor = (
|
||||
_prefer_two_stage_compressor() and head_dim == 512 and not self.overlap
|
||||
)
|
||||
self.max_num_batched_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self._compress_scratch: torch.Tensor | None = None
|
||||
if self._use_two_stage_fused_compressor:
|
||||
self._compress_scratch = torch.empty(
|
||||
self.max_num_batched_tokens,
|
||||
self.head_dim,
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
state_dtype = torch.float32
|
||||
self.ape = nn.Parameter(
|
||||
torch.empty(
|
||||
(compress_ratio, self.coff * self.head_dim),
|
||||
dtype=state_dtype,
|
||||
device=self.device,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.fused_wkv_wgate = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim],
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
disable_tp=True,
|
||||
prefix=f"{prefix}.fused_wkv_wgate",
|
||||
)
|
||||
self.norm = RMSNorm(self.head_dim, self.rms_norm_eps)
|
||||
|
||||
self.state_cache = CompressorStateCache(
|
||||
state_dim=2 * self.coff * self.head_dim, # kv_state + score_state
|
||||
dtype=state_dtype,
|
||||
compress_ratio=compress_ratio,
|
||||
prefix=f"{prefix}.state_cache",
|
||||
)
|
||||
|
||||
# Save reference to static_forward_context for forward-time KV cache lookup.
|
||||
# get_current_vllm_config() is only available during __init__, not forward.
|
||||
self._static_forward_context = (
|
||||
vllm_config.compilation_config.static_forward_context
|
||||
)
|
||||
|
||||
if self.head_dim == 512:
|
||||
assert not use_fp4_cache, (
|
||||
"MXFP4 cache is only supported for indexer (head=128)"
|
||||
)
|
||||
self._quant_block = 64
|
||||
self._token_stride = self.nope_head_dim + self.rope_head_dim * 2
|
||||
self._scale_dim = self.nope_head_dim // 64 + 1 # 7 real + 1 pad
|
||||
elif self.head_dim == 128:
|
||||
if use_fp4_cache:
|
||||
self._quant_block = MXFP4_BLOCK_SIZE
|
||||
self._token_stride = self.head_dim // 2
|
||||
self._scale_dim = self.head_dim // MXFP4_BLOCK_SIZE
|
||||
else:
|
||||
self._quant_block = 128
|
||||
self._token_stride = self.head_dim
|
||||
self._scale_dim = 4 # single float32 scale
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_SAVE_PARTIAL_STATES_KERNEL.register_warmup(
|
||||
head_dim=self.head_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
if current_platform.is_cuda() and self.head_dim == 512:
|
||||
from vllm.models.deepseek_v4.nvidia.ops.sparse_attn_compress_cutedsl import ( # noqa: E501
|
||||
_SPARSE_ATTN_COMPRESS_C128_BLOCK8_KERNEL,
|
||||
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_C4_KERNEL,
|
||||
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_FULL_C4_KERNEL,
|
||||
_SPARSE_ATTN_NORM_ROPE_STORE_FULL_KERNEL,
|
||||
_SPARSE_ATTN_NORM_ROPE_STORE_KERNEL,
|
||||
)
|
||||
|
||||
store_full_kv = vllm_config.cache_config.cache_dtype != "fp8_ds_mla"
|
||||
if self.compress_ratio == 4:
|
||||
(
|
||||
_SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_FULL_C4_KERNEL
|
||||
if store_full_kv
|
||||
else _SPARSE_ATTN_COMPRESS_NORM_ROPE_STORE_C4_KERNEL
|
||||
).register_warmup()
|
||||
else:
|
||||
_SPARSE_ATTN_COMPRESS_C128_BLOCK8_KERNEL.register_warmup()
|
||||
if store_full_kv:
|
||||
_SPARSE_ATTN_NORM_ROPE_STORE_FULL_KERNEL.register_warmup()
|
||||
else:
|
||||
_SPARSE_ATTN_NORM_ROPE_STORE_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
k_cache_prefix=self.k_cache_prefix,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
else:
|
||||
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( # noqa: E501
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL,
|
||||
)
|
||||
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
# [num_tokens, 2 * self.coff * self.head_dim]
|
||||
kv_score: torch.Tensor,
|
||||
# [num_tokens]
|
||||
positions: torch.Tensor,
|
||||
rotary_emb,
|
||||
) -> None:
|
||||
# Each of shape [num_tokens, coff * self.head_dim]
|
||||
# input bf16, output are fp32
|
||||
kv, score = kv_score.split(
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
|
||||
)
|
||||
|
||||
# Get the metadata and handle dummy profiling run.
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
|
||||
state_metadata = cast(
|
||||
CompressorMetadata, attn_metadata[self.state_cache.prefix]
|
||||
)
|
||||
token_to_req_indices = state_metadata.token_to_req_indices
|
||||
slot_mapping = state_metadata.slot_mapping
|
||||
num_actual = slot_mapping.shape[0]
|
||||
block_table = state_metadata.block_table
|
||||
block_size = state_metadata.block_size
|
||||
|
||||
# [num_blocks, block_size, kv_dim+score_dim], where kv_dim == score_dim
|
||||
state_cache = self.state_cache.kv_cache
|
||||
# kv_state stored in first half, score_state stored in second half
|
||||
state_width = state_cache.shape[-1] // 2
|
||||
pdl_kwargs = (
|
||||
{}
|
||||
if current_platform.is_rocm() or current_platform.is_xpu()
|
||||
else {"launch_pdl": False}
|
||||
)
|
||||
|
||||
# Store the KV and score (with fused APE addition) in the state.
|
||||
# NOTE: PDL is disabled — both this kernel and the compress kernels
|
||||
# below depend on preceding kernel outputs (kv/score from the cublas
|
||||
# GEMM; state_cache from this kernel) but neither emits/waits on PDL
|
||||
# grid dependency primitives, so launch_pdl=True caused a
|
||||
# read-after-write race and non-deterministic output.
|
||||
_SAVE_PARTIAL_STATES_KERNEL(
|
||||
kv=kv,
|
||||
score=score,
|
||||
ape=self.ape,
|
||||
positions=positions,
|
||||
state_cache=state_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
block_size=block_size,
|
||||
state_width=state_width,
|
||||
compress_ratio=self.compress_ratio,
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
)
|
||||
|
||||
# A captured graph would replay the capture batch's skip decision.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and self.head_dim == 512
|
||||
and self.compress_ratio == 128
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
and state_metadata.c128_boundary is False
|
||||
):
|
||||
return
|
||||
|
||||
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
|
||||
# RoPE requirements (kernel applies forward GPT-J style rotation):
|
||||
# - is_neox_style=False (interleaved pairs, NOT split-half)
|
||||
# - cos_sin_cache layout: [max_pos, rope_head_dim] with first half cos,
|
||||
# second half sin (per-pair, length rope_head_dim // 2 each)
|
||||
# - applied to LAST rope_head_dim elements of head_dim
|
||||
# - position used: (positions // compress_ratio) * compress_ratio
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
# Plain-row V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# fp8_ds_mla path uses the UE8M0 paged uint8 layout.
|
||||
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
|
||||
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
|
||||
fp8_scale = (
|
||||
getattr(k_cache_layer, "_flashinfer_fp8_kv_scale", None)
|
||||
if store_full_fp8
|
||||
else None
|
||||
)
|
||||
|
||||
# cutedsl (head=512) accepts the full-cache flags; triton (indexer/AMD)
|
||||
# does not, so the two callables have different signatures.
|
||||
compress_norm_rope_store_fn: Any
|
||||
if current_platform.is_cuda() and self.head_dim == 512:
|
||||
from .nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
_SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL,
|
||||
)
|
||||
|
||||
# head=512 on CUDA always uses cutedsl, for both the fp8_ds_mla
|
||||
# layout and the plain full-cache layout. The full-cache flags
|
||||
# are consumed only here.
|
||||
compress_norm_rope_store_fn = _SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL
|
||||
extra_kwargs: dict[str, Any] = dict(
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
elif self._use_two_stage_fused_compressor:
|
||||
# head=512 cr>=128 (no overlap): two-pass split compressor on the
|
||||
# prefill suffix, single-pass on the decode prefix.
|
||||
assert state_metadata.num_decode_tokens is not None
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_two_stage_triton
|
||||
extra_kwargs = {
|
||||
"num_decode_tokens": state_metadata.num_decode_tokens,
|
||||
"compress_scratch": self._compress_scratch,
|
||||
}
|
||||
else:
|
||||
# Indexer path (head_dim == 128) or non-CUDA GPUs (AMD, XPU, etc.).
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
extra_kwargs = {}
|
||||
|
||||
compress_norm_rope_store_fn(
|
||||
state_cache=state_cache,
|
||||
num_actual=num_actual,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
positions=positions,
|
||||
slot_mapping=slot_mapping,
|
||||
block_table=block_table,
|
||||
block_size=block_size,
|
||||
state_width=state_width,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
kv_cache=kv_cache,
|
||||
k_cache_metadata=k_cache_metadata,
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
head_dim=self.head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
overlap=self.overlap,
|
||||
use_fp4_cache=self.use_fp4_cache,
|
||||
rms_norm_weight=self.norm.weight,
|
||||
rms_norm_eps=self.rms_norm_eps,
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
**extra_kwargs,
|
||||
)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 compressor subclass and dispatch: partial-state-cache
|
||||
write and the compress -> RMSNorm -> RoPE -> FP8 quant -> KV cache store
|
||||
step, calling ``torch.ops._C.save_partial_states_cpu``/
|
||||
``compress_norm_rope_store_cpu``/``compress_norm_rope_store_indexer_cpu``
|
||||
(``csrc/cpu/sgl-kernels/compressor.cpp``) in place of triton-cpu.
|
||||
|
||||
Covers both the head_dim=512 main-attention compressor path and the
|
||||
head_dim=128 indexer compressor path (fp8 only -- MXFP4, ``use_fp4_cache``,
|
||||
never occurs on CPU: the indexer's MXFP4 cache requires a Blackwell GPU).
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import (
|
||||
compress_norm_rope_store_cpu,
|
||||
compress_norm_rope_store_indexer_cpu,
|
||||
save_partial_states_cpu,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.compressor import CompressorMetadata, DeepseekCompressor
|
||||
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
|
||||
|
||||
|
||||
class DeepseekV4CPUCompressor(DeepseekCompressor):
|
||||
"""CPU compressor: same state/weights as the shared base, but
|
||||
``forward``'s cache-write and compress->RMSNorm->RoPE->quant->store
|
||||
steps always dispatch straight to the ported CPU kernels above instead
|
||||
of through the shared method's platform-dispatch chain. This class
|
||||
only ever runs on CPU, and ``use_fp4_cache`` is always ``False`` here
|
||||
(the indexer's MXFP4 cache requires a Blackwell GPU), so the shared
|
||||
method's MXFP4/two-stage/cutedsl/triton branches never apply.
|
||||
"""
|
||||
|
||||
_norm_weight_fp32: torch.Tensor
|
||||
|
||||
def cache_norm_weight_fp32(self) -> None:
|
||||
"""Cache a fp32 contiguous copy of ``norm.weight`` once, right after
|
||||
weight loading (see
|
||||
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
|
||||
kernel requires fp32/contiguous but the checkpoint loads this weight
|
||||
in bf16, and it never changes after loading."""
|
||||
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
kv_score: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb,
|
||||
) -> None:
|
||||
kv, score = kv_score.split(
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
|
||||
)
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
|
||||
state_metadata = attn_metadata[self.state_cache.prefix]
|
||||
assert isinstance(state_metadata, CompressorMetadata)
|
||||
token_to_req_indices = state_metadata.token_to_req_indices
|
||||
assert token_to_req_indices is not None
|
||||
slot_mapping = state_metadata.slot_mapping
|
||||
num_actual = slot_mapping.shape[0]
|
||||
block_table = state_metadata.block_table
|
||||
block_size = state_metadata.block_size
|
||||
|
||||
state_cache = self.state_cache.kv_cache
|
||||
|
||||
save_partial_states_cpu(
|
||||
kv, score, self.ape, positions, state_cache, slot_mapping
|
||||
)
|
||||
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
assert self.head_dim in (512, 128), (
|
||||
f"unsupported compressor head_dim: {self.head_dim}"
|
||||
)
|
||||
window = (1 + self.overlap) * self.compress_ratio
|
||||
win_offsets = torch.arange(window, device=positions.device)
|
||||
win_local = positions.unsqueeze(-1) - window + 1 + win_offsets
|
||||
req_idx = token_to_req_indices[:num_actual]
|
||||
gather_slots = map_local_to_global_slots_cpu(
|
||||
win_local, req_idx, block_table, block_size
|
||||
)
|
||||
|
||||
kv_cache_2d = kv_cache.view(kv_cache.shape[0], -1)
|
||||
kv_cache_block_size = kv_cache.shape[1]
|
||||
|
||||
store_fn = (
|
||||
compress_norm_rope_store_cpu
|
||||
if self.head_dim == 512
|
||||
else compress_norm_rope_store_indexer_cpu
|
||||
)
|
||||
store_fn(
|
||||
state_cache,
|
||||
gather_slots,
|
||||
positions,
|
||||
k_cache_metadata.slot_mapping,
|
||||
self._norm_weight_fp32,
|
||||
self.rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
kv_cache_2d,
|
||||
kv_cache_block_size,
|
||||
self.compress_ratio,
|
||||
)
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 sparse-MLA backend descriptor.
|
||||
|
||||
DeepSeek-V4 attention runs entirely through
|
||||
``DeepseekV4Attention``/``DeepseekV4CPUAttention``, never through a generic
|
||||
``AttentionImpl``. ``DeepseekV4CPUSparseBackend`` exists so the CPU attention
|
||||
layer has its own backend name and its own metadata-builder class: CPU is not
|
||||
allowed to run triton-cpu inside the model's forward path, so
|
||||
``DeepseekV4CPUFlashMLAMetadataBuilder`` overrides the shared base's
|
||||
``_build_c128a_metadata`` (triton-backed in
|
||||
``DeepseekV4SparseMLAMetadataBuilder``) with a plain-eager-PyTorch
|
||||
reimplementation of the same per-token block-table-resolution math.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend,
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_LAYER_TYPE_C4A,
|
||||
_LAYER_TYPE_C128A,
|
||||
_LAYER_TYPE_SWAONLY,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta
|
||||
|
||||
|
||||
class DeepseekV4CPUFlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
"""CPU sparse-MLA metadata builder: same fields as the shared base, but
|
||||
skips the C128A dense-topk metadata entirely -- unlike CUDA/XPU/ROCm,
|
||||
``DeepseekV4CPUAttention.forward_mqa`` never reads
|
||||
``c128a_global_decode_topk_indices``/``c128a_decode_topk_lens``/
|
||||
``c128a_prefill_topk_indices``; it recomputes the same local top-k
|
||||
directly from ``positions`` instead, so building it here (whether
|
||||
eagerly or via the shared base's triton kernel) would be wasted work.
|
||||
"""
|
||||
|
||||
def _build_c128a_metadata(
|
||||
self,
|
||||
cm: CommonAttentionMetadata,
|
||||
req_id_per_token: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
return {}
|
||||
|
||||
|
||||
class DeepseekV4CPUSparseBackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_V4_MLA_SPARSE"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4CPUFlashMLAMetadataBuilder]:
|
||||
return DeepseekV4CPUFlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder):
|
||||
"""CPU indexer metadata builder: same fields as the shared base, but
|
||||
prefill requests are never split into multiple chunks.
|
||||
|
||||
``DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks`` (the
|
||||
shared base's default) bounds two things the CUDA/XPU indexer kernels
|
||||
need: the flat-gather workspace size and the dense M*N logits tensor
|
||||
those kernels allocate for a chunk. The CPU indexer
|
||||
(``sparse_attn_indexer_cpu``) allocates neither -- it reads the paged
|
||||
K-cache directly via ``fp8_paged_mqa_logits_cpu``/
|
||||
``topk_transform_512_cpu``, with no chunk-splitting of its own.
|
||||
With chunked prefill now enabled for this model on CPU (see
|
||||
``CpuPlatform.check_and_update_config``'s ``amx_mla_or_dsv4_enabled``),
|
||||
the total query-token count for one step is already bounded by
|
||||
``max_num_batched_tokens`` -- there is nothing left here to bound, so
|
||||
this always returns a single chunk spanning the whole step's prefill
|
||||
batch.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _split_indexer_prefill_chunks(
|
||||
compressed_seq_lens_cpu: torch.Tensor,
|
||||
prefill_query_lens_cpu: torch.Tensor,
|
||||
workspace_size: int,
|
||||
max_logits_bytes: int,
|
||||
request_offset: int = 0,
|
||||
) -> list[tuple[slice, slice]]:
|
||||
# workspace_size/max_logits_bytes are unused: no flat-gather
|
||||
# workspace or dense M*N buffer is allocated.
|
||||
num_reqs = compressed_seq_lens_cpu.shape[0]
|
||||
total_query_len = int(prefill_query_lens_cpu.sum().item())
|
||||
return [
|
||||
(
|
||||
slice(request_offset, request_offset + num_reqs),
|
||||
slice(0, total_query_len),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerBackend(DeepseekV4IndexerBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_V4_INDEXER"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4CPUIndexerMetadataBuilder]:
|
||||
return DeepseekV4CPUIndexerMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
"""CPU SWA metadata builder: same fields as the shared base."""
|
||||
|
||||
def build_tile_scheduler(
|
||||
self, num_decode_tokens: int
|
||||
) -> dict[str, FlashMLASchedMeta | None]:
|
||||
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
|
||||
C++ decode-path concern) -- always return the all-``None`` sentinel
|
||||
the shared base's own CPU branch returns."""
|
||||
return {
|
||||
_LAYER_TYPE_SWAONLY: None,
|
||||
_LAYER_TYPE_C4A: None,
|
||||
_LAYER_TYPE_C128A: None,
|
||||
}
|
||||
@@ -1,649 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 attention subclass.
|
||||
|
||||
``forward_mqa`` resolves SWA/compressed top-k indices to paged-cache slot
|
||||
ids and calls the fused ``flash_mla_with_kvcache_cpu`` kernel. ``_o_proj``
|
||||
stays eager -- not on the attention hot path.
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4Attention,
|
||||
DeepseekV4Indexer,
|
||||
DeepseekV4IndexerCache,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_compressor import DeepseekV4CPUCompressor
|
||||
from vllm.models.deepseek_v4.cpu.cpu_mla import (
|
||||
DeepseekV4CPUIndexerBackend,
|
||||
DeepseekV4CPUSparseBackend,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_ROPE_DIM = 64
|
||||
|
||||
|
||||
def _deepseek_v4_cpu_prepare_and_attn(
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor | None,
|
||||
indexer_kv_score: torch.Tensor | None,
|
||||
indexer_weights: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""Opaque custom-op boundary around attention-prep + sparse-indexer +
|
||||
MLA attention + output projection -- CPU-only.
|
||||
|
||||
Wraps cache writes with no explicit tensor args (SWA/compressor/indexer
|
||||
state), which torch.compile can't otherwise track as mutations -- one
|
||||
opaque call sidesteps that. ``_o_proj`` is folded in too because CPU's
|
||||
``DYNAMO_TRACE_ONCE`` mode traces exactly once, during warmup when
|
||||
``attn_metadata`` is ``None``; tracing ``_o_proj`` separately would bake
|
||||
that warmup branch into the compiled graph permanently, dead-code-
|
||||
eliminating the whole attention computation upstream.
|
||||
"""
|
||||
layer_name_str = _resolve_layer_name(layer_name)
|
||||
self = get_forward_context().no_compile_layers[layer_name_str]
|
||||
self._prepare_and_attn_fn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
|
||||
def _deepseek_v4_cpu_prepare_and_attn_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor | None,
|
||||
indexer_kv_score: torch.Tensor | None,
|
||||
indexer_weights: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
# hidden_states' last dim already equals _o_proj's output width, so we
|
||||
# don't need to resolve layer_name (an opaque object) here.
|
||||
return hidden_states.new_empty((hidden_states.shape[0], hidden_states.shape[-1]))
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="deepseek_v4_cpu_prepare_and_attn",
|
||||
op_func=_deepseek_v4_cpu_prepare_and_attn,
|
||||
mutates_args=["o_padded"],
|
||||
fake_impl=_deepseek_v4_cpu_prepare_and_attn_fake,
|
||||
)
|
||||
|
||||
|
||||
def _dequant_linear_weight(layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Dequantize a linear layer's weight, including FP8 block scales.
|
||||
|
||||
``_o_proj`` reads ``wo_a.weight`` directly (not via
|
||||
``quant_method.apply()``), so the per-block FP8 scale must be applied
|
||||
here explicitly.
|
||||
"""
|
||||
weight = layer.weight
|
||||
scale = getattr(layer, "weight_scale_inv", None)
|
||||
if scale is None:
|
||||
scale = getattr(layer, "weight_scale", None)
|
||||
if scale is None:
|
||||
return weight.float()
|
||||
|
||||
if scale.dtype == torch.float8_e8m0fnu:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
scale = _upcast_e8m0_to_fp32(scale)
|
||||
scale = scale.to(torch.float32)
|
||||
|
||||
if scale.numel() == 1:
|
||||
return weight.float() * scale
|
||||
|
||||
block_size = getattr(layer, "weight_block_size", None)
|
||||
if block_size is None:
|
||||
# Per-channel scale: one row-scale per output row.
|
||||
return weight.float() * scale.view(-1, 1)
|
||||
|
||||
block_m, block_k = block_size
|
||||
m, k = weight.shape
|
||||
scale = scale.repeat_interleave(block_m, dim=0).repeat_interleave(block_k, dim=1)
|
||||
return weight.float() * scale[:m, :k]
|
||||
|
||||
|
||||
def _fused_indexer_q_rope_quant_cpu(
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""CPU-only equivalent of ``fused_indexer_q_rope_quant``'s FP8 path
|
||||
(CPU never reaches its MXFP4 arm -- Blackwell-only). Returns
|
||||
``(q_fp8, weights_out)``, folding the per-token q scale into
|
||||
``index_weights``. ``positions``/``index_q_cos_sin_cache`` are already
|
||||
contiguous int64/fp32, so no conversion is needed here.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert index_q.ndim == 3
|
||||
assert index_q_cos_sin_cache.ndim == 2
|
||||
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=current_platform.fp8_dtype())
|
||||
ops.fused_indexer_q_rope_quant_cpu(
|
||||
positions,
|
||||
index_q.to(torch.float32),
|
||||
index_q_cos_sin_cache,
|
||||
index_q_fp8,
|
||||
index_weights.to(torch.float32),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
return index_q_fp8, index_weights_out
|
||||
|
||||
|
||||
class _NoOpEvent:
|
||||
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
|
||||
stream list, these events are constructed but never actually used."""
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
|
||||
"""CPU indexer K-cache descriptor: same fields as the shared base, just
|
||||
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV4CPUIndexerBackend
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexer(DeepseekV4Indexer):
|
||||
"""CPU indexer: the C4A short-context fallback runs as eager PyTorch
|
||||
instead of Triton. Never constructed directly -- ``__class__`` is
|
||||
swapped onto an existing ``DeepseekV4Indexer`` instance post-construction.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
compressed_kv_score: torch.Tensor | None,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
qr_scale: torch.Tensor | None = None,
|
||||
skip_compressor: bool = False,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""CPU override: no aux streams, so wq_b_and_q_quant and the
|
||||
compressor run straight-line instead of through
|
||||
``maybe_execute_in_parallel``; the short-context check drops the
|
||||
CUDA-only ``is_current_stream_capturing()`` guard (always False
|
||||
here)."""
|
||||
compressor = self.compressor
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
|
||||
if indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens:
|
||||
# Fewer candidates than topk: all are selected, but the K
|
||||
# cache still needs to be built.
|
||||
compressor(compressed_kv_score, positions, rotary_emb)
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_tokens = (
|
||||
indexer_metadata.num_decode_tokens
|
||||
+ indexer_metadata.num_prefill_tokens
|
||||
)
|
||||
if num_tokens > 0:
|
||||
self._fill_short_context_topk(positions, num_tokens)
|
||||
return None, None, None
|
||||
|
||||
def wq_b_and_q_quant():
|
||||
q = self._wq_b_proj(qr, qr_scale)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
return _fused_indexer_q_rope_quant_cpu(
|
||||
positions,
|
||||
q,
|
||||
rotary_emb.cos_sin_cache,
|
||||
indexer_weights,
|
||||
self.softmax_scale,
|
||||
self.n_head**-0.5,
|
||||
)
|
||||
|
||||
# Order matches the base class's parallel-join contract: compressor
|
||||
# must complete before indexer_op runs (skip_k_cache_insert=True).
|
||||
q, weights = wq_b_and_q_quant()
|
||||
compressor(compressed_kv_score, positions, rotary_emb)
|
||||
# No q_scale: FP8-only path (MXFP4 needs Blackwell).
|
||||
return q, None, weights
|
||||
|
||||
def _fill_short_context_topk(
|
||||
self, positions: torch.Tensor, num_tokens: int
|
||||
) -> None:
|
||||
assert self.topk_indices_buffer is not None
|
||||
device = positions.device
|
||||
arange = torch.arange(self.topk_tokens, device=device)
|
||||
num_compressed = torch.div(
|
||||
positions[:num_tokens] + 1, self.compress_ratio, rounding_mode="floor"
|
||||
)
|
||||
valid = arange.unsqueeze(0) < num_compressed.unsqueeze(-1)
|
||||
filled = torch.where(
|
||||
valid, arange.unsqueeze(0), torch.full_like(arange.unsqueeze(0), -1)
|
||||
)
|
||||
self.topk_indices_buffer[:num_tokens, : self.topk_tokens] = filled.to(
|
||||
self.topk_indices_buffer.dtype
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
"""CPU sparse MLA attention layer for DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4CPUSparseBackend
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# Swap classes post-construction (no class-swap hook on the shared
|
||||
# base) so forward()/get_attn_backend() dispatch to CPU kernels.
|
||||
# SWA-only layers (compress_ratio <= 1) never build a
|
||||
# compressor/indexer.
|
||||
if self.compressor is not None:
|
||||
self.compressor.__class__ = DeepseekV4CPUCompressor
|
||||
if self.indexer is not None:
|
||||
self.indexer.__class__ = DeepseekV4CPUIndexer
|
||||
self.indexer.k_cache.__class__ = DeepseekV4CPUIndexerCache
|
||||
self.indexer.compressor.__class__ = DeepseekV4CPUCompressor
|
||||
|
||||
self._wo_a_packed: torch.Tensor | None = None
|
||||
self._wrap_wo_a_process_weights_after_loading()
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
"""Cache fp32-contiguous copies of tensors CPU kernels want that way
|
||||
but that arrive bf16 (rotary cos/sin table, compressor RMSNorm
|
||||
weights) -- cast once here instead of per forward call.
|
||||
|
||||
Runs after every quantized layer's own
|
||||
``process_weights_after_loading`` (see ``is_deferred_attention_layer``);
|
||||
``wo_a``'s packing must happen *before* that phase instead, so it's a
|
||||
separate monkeypatch in ``__init__``.
|
||||
"""
|
||||
self.rotary_emb.cos_sin_cache = self.rotary_emb.cos_sin_cache.to(
|
||||
torch.float32
|
||||
).contiguous()
|
||||
if self.compressor is not None:
|
||||
self.compressor.cache_norm_weight_fp32()
|
||||
if self.indexer is not None:
|
||||
self.indexer.compressor.cache_norm_weight_fp32()
|
||||
|
||||
def _wrap_wo_a_process_weights_after_loading(self) -> None:
|
||||
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
|
||||
``bmm_cpu`` before the FP8 kernel's own
|
||||
``process_weights_after_loading`` VNNI-repacks it in place for
|
||||
row-major reads ``_o_proj`` never does."""
|
||||
wo_a = self.wo_a
|
||||
quant_method = wo_a.quant_method
|
||||
orig_pwal = quant_method.process_weights_after_loading
|
||||
heads_per_group = self.n_local_heads // self.n_local_groups
|
||||
k_dim = heads_per_group * self.head_dim
|
||||
|
||||
def _capture_and_pack(layer: torch.nn.Module) -> None:
|
||||
dequant = _dequant_linear_weight(layer).to(torch.bfloat16)
|
||||
dequant = dequant.view(self.n_local_groups, self.o_lora_rank, k_dim)
|
||||
self._wo_a_packed = torch.ops._C.convert_weight_packed(dequant.contiguous())
|
||||
orig_pwal(layer)
|
||||
# orig_pwal VNNI-repacks layer.weight/scale for ordinary-linear
|
||||
# use that _o_proj never needs -- free them, same pattern as
|
||||
# dispatch_cpu_unquantized_gemm's remove_weight=True.
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
scale_attr = (
|
||||
"weight_scale_inv"
|
||||
if hasattr(layer, "weight_scale_inv")
|
||||
else "weight_scale"
|
||||
)
|
||||
if hasattr(layer, scale_attr):
|
||||
setattr(
|
||||
layer,
|
||||
scale_attr,
|
||||
torch.nn.Parameter(torch.empty(0), requires_grad=False),
|
||||
)
|
||||
|
||||
quant_method.process_weights_after_loading = ( # type: ignore[method-assign]
|
||||
_capture_and_pack
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: wraps attention-prep + MLA + output projection in the
|
||||
CPU-only opaque custom op (see ``_deepseek_v4_cpu_prepare_and_attn``).
|
||||
"""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
o_padded = torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
|
||||
|
||||
kv_score: torch.Tensor | None = None
|
||||
if self.compressor is not None:
|
||||
# Must go through the module's own forward (not a raw matmul on
|
||||
# its weight): the packed-GEMM kernel frees the raw weight after
|
||||
# packing, same as wo_a's.
|
||||
kv_score = self.compressor.fused_wkv_wgate(hidden_states).to(torch.float32)
|
||||
|
||||
indexer_weights: torch.Tensor | None = None
|
||||
indexer_kv_score: torch.Tensor | None = None
|
||||
if self.indexer is not None:
|
||||
indexer_weights, _ = self.indexer.weights_proj(hidden_states)
|
||||
indexer_kv_score = self.indexer.compressor.fused_wkv_wgate(
|
||||
hidden_states
|
||||
).to(torch.float32)
|
||||
|
||||
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
||||
|
||||
return torch.ops.vllm.deepseek_v4_cpu_prepare_and_attn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def _fused_qnorm_rope_kv_insert(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
attn_metadata: (
|
||||
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
|
||||
),
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
|
||||
supported here, so the base method's bf16/per-tensor-fp8 branches
|
||||
are dropped."""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
return q
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
# positions is already int64 (the runner's buffer), no cast needed.
|
||||
assert positions.dtype == torch.int64
|
||||
assert swa_kv_cache.dtype == torch.uint8, (
|
||||
"DeepseekV4CPUAttention only supports the fp8_ds_mla cache layout"
|
||||
)
|
||||
# Fused: Q side does per-head weight-free RMSNorm + GPT-J RoPE
|
||||
# (zero-filling padded head slots); KV side does GPT-J RoPE + UE8M0
|
||||
# FP8 quant + paged insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
return ops.fused_qnorm_rope_kv_insert_cpu(
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.padded_heads,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""CPU override: two ``RMSNorm`` calls instead of the shared
|
||||
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
return self.q_norm(qr), None, self.kv_norm(kv)
|
||||
|
||||
def _prepare_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: no aux streams, so query-projection+KV-insert, the
|
||||
indexer, and the compressor run straight-line instead of through
|
||||
execute_in_parallel/maybe_execute_in_parallel."""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
|
||||
q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_local_heads, self.head_dim)
|
||||
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
index_q: torch.Tensor | None = None
|
||||
index_q_scale: torch.Tensor | None = None
|
||||
index_weights_out: torch.Tensor | None = None
|
||||
if indexer is not None:
|
||||
assert compressor is not None
|
||||
index_q, index_q_scale, index_weights_out = indexer(
|
||||
hidden_states,
|
||||
qr,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
self.indexer_rotary_emb,
|
||||
qr_scale,
|
||||
)
|
||||
compressor(kv_score, positions, self.rotary_emb)
|
||||
elif compressor is not None:
|
||||
compressor(kv_score, positions, self.rotary_emb)
|
||||
|
||||
self._sparse_indexer_and_attn(
|
||||
hidden_states,
|
||||
index_q,
|
||||
index_q_scale,
|
||||
index_weights_out,
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
|
||||
def _sparse_indexer_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
index_q: torch.Tensor | None,
|
||||
index_q_scale: torch.Tensor | None,
|
||||
index_weights: torch.Tensor | None,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: identical body, minus ``@eager_break_during_capture``
|
||||
(this platform never captures a CUDA graph, so it was a no-op)."""
|
||||
if self.indexer is not None and index_q is not None:
|
||||
assert index_weights is not None
|
||||
# index_q_scale is always None on CPU (FP8-only path).
|
||||
assert index_q_scale is None
|
||||
self.indexer.indexer_op(hidden_states, index_q, None, index_weights)
|
||||
|
||||
self.forward_mqa(q, kv, positions, out)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# No head-count padding constraint for the (not yet ported) CPU kernel.
|
||||
return num_heads
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
if get_forward_context().attn_metadata is None:
|
||||
# Warmup/profile run: no real metadata, o is a zeroed placeholder.
|
||||
return torch.zeros(
|
||||
(o.shape[0], self.hidden_size), dtype=o.dtype, device=o.device
|
||||
)
|
||||
|
||||
num_tokens, num_heads, _ = o.shape
|
||||
cos_sin_cache = self.rotary_emb.cos_sin_cache
|
||||
# Undo the query's own GPT-J RoPE rotation: the attention output is
|
||||
# a weighted sum of KV rows at different positions, so R(-pos_q)
|
||||
# recovers the translation-invariant quantity wo_a/wo_b expect.
|
||||
o_derot = ops.inverse_gptj_rope_o_proj_cpu(
|
||||
o, positions, cos_sin_cache, _ROPE_DIM
|
||||
)
|
||||
|
||||
heads_per_group = num_heads // self.n_local_groups
|
||||
o_grouped = o_derot.reshape(
|
||||
num_tokens, self.n_local_groups, heads_per_group * self.head_dim
|
||||
).to(torch.bfloat16)
|
||||
assert self._wo_a_packed is not None
|
||||
z = o_grouped.new_empty((self.n_local_groups, num_tokens, self.o_lora_rank))
|
||||
ops.bmm_cpu(
|
||||
z,
|
||||
o_grouped.transpose(0, 1).contiguous(),
|
||||
self._wo_a_packed,
|
||||
True,
|
||||
None,
|
||||
)
|
||||
z = z.transpose(0, 1).to(o.dtype)
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup/profile run: reserve nothing extra, skip real kernels.
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
swa_metadata = cast(
|
||||
DeepseekSparseSWAMetadata,
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
assert swa_metadata.token_to_req_indices is not None
|
||||
req_idx = swa_metadata.token_to_req_indices
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
|
||||
num_tokens = q.shape[0]
|
||||
device = q.device
|
||||
|
||||
# ---- SWA window: causal sliding window over the raw per-token cache
|
||||
window = self.window_size
|
||||
win_offsets = torch.arange(window, device=device)
|
||||
start_pos = (positions - window + 1).clamp(min=0)
|
||||
local_pos = start_pos.unsqueeze(-1) + win_offsets
|
||||
valid_win = local_pos <= positions.unsqueeze(-1)
|
||||
win_local = torch.where(valid_win, local_pos, torch.full_like(local_pos, -1))
|
||||
window_slots = map_local_to_global_slots_cpu(
|
||||
win_local, req_idx, swa_metadata.block_table, swa_metadata.block_size
|
||||
)
|
||||
window_cache_2d = self.swa_cache_layer.kv_cache.view(
|
||||
self.swa_cache_layer.kv_cache.shape[0], -1
|
||||
)
|
||||
|
||||
# ---- Compressed KV: C4A's learned top-k, or C128A's dense top-k ----
|
||||
if swa_only:
|
||||
compressed_slots = window_slots.new_full((num_tokens, 0), -1)
|
||||
# Unused when num_compressed == 0 (only dtype/2D-ness matter);
|
||||
# reuse the window cache to avoid an allocation.
|
||||
compressed_cache_2d = window_cache_2d
|
||||
compressed_block_size = 1
|
||||
else:
|
||||
assert flashmla_metadata is not None
|
||||
main_block_size = flashmla_metadata.block_size // self.compress_ratio
|
||||
if self.compress_ratio == 4:
|
||||
assert self.topk_indices_buffer is not None
|
||||
local_topk = self.topk_indices_buffer[:num_tokens]
|
||||
else:
|
||||
width = -(-self.max_model_len // self.compress_ratio)
|
||||
num_compressed = torch.div(
|
||||
positions + 1, self.compress_ratio, rounding_mode="floor"
|
||||
).clamp(max=width)
|
||||
arange_w = torch.arange(width, device=device)
|
||||
local_topk = torch.where(
|
||||
arange_w.unsqueeze(0) < num_compressed.unsqueeze(-1),
|
||||
arange_w.unsqueeze(0).expand(num_tokens, width),
|
||||
torch.full(
|
||||
(num_tokens, width), -1, device=device, dtype=torch.int64
|
||||
),
|
||||
)
|
||||
compressed_slots = map_local_to_global_slots_cpu(
|
||||
local_topk, req_idx, flashmla_metadata.block_table, main_block_size
|
||||
)
|
||||
compressed_cache_2d = self.kv_cache.view(self.kv_cache.shape[0], -1)
|
||||
compressed_block_size = main_block_size
|
||||
|
||||
torch.ops._C.flash_mla_with_kvcache_cpu(
|
||||
output,
|
||||
q,
|
||||
window_cache_2d,
|
||||
window_slots,
|
||||
swa_metadata.block_size,
|
||||
compressed_cache_2d,
|
||||
compressed_slots,
|
||||
compressed_block_size,
|
||||
self.attn_sink,
|
||||
self.scale,
|
||||
)
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for DeepSeek-V4's CPU-ported kernels."""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def map_local_to_global_slots_cpu(
|
||||
local_indices: torch.Tensor,
|
||||
req_idx: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map per-request-local (token or compressed-token) indices to global
|
||||
paged-cache slot ids via a block table, mirroring
|
||||
``compute_global_topk_indices_and_lens``.
|
||||
|
||||
Args:
|
||||
local_indices: [N, K], -1 sentinel for invalid entries.
|
||||
req_idx: [N], row into ``block_table`` for each of the N rows.
|
||||
block_table: [num_reqs, max_blocks_per_seq].
|
||||
block_size: tokens (or compressed tokens) per physical block.
|
||||
|
||||
Returns:
|
||||
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
|
||||
|
||||
"""
|
||||
valid = local_indices >= 0
|
||||
safe_local = local_indices.clamp(min=0).to(torch.int64)
|
||||
block_pos = torch.div(safe_local, block_size, rounding_mode="floor")
|
||||
offset = safe_local % block_size
|
||||
req_expand = req_idx.to(torch.int64).unsqueeze(-1).expand_as(local_indices)
|
||||
block_num = block_table[req_expand, block_pos].to(torch.int64)
|
||||
slot = block_num * block_size + offset
|
||||
return torch.where(valid, slot, torch.full_like(slot, -1))
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 DSpark (speculative decoding) — deferred, not implemented.
|
||||
|
||||
``DSparkDeepseekV4ForCausalLM`` is only instantiated when the speculative-
|
||||
decoding registry resolves it by name for a DSpark speculative config; a
|
||||
plain (non-spec) DeepSeek-V4 CPU model never constructs this class. This
|
||||
stub exists solely so ``vllm.models.deepseek_v4.cpu`` is import-clean.
|
||||
"""
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM:
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 speculative decoding (DSpark) is not implemented on CPU."
|
||||
)
|
||||
@@ -1,900 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_TILELANG_MHC,
|
||||
HCHeadOp,
|
||||
MHCFusedPostPreOp,
|
||||
MHCPostOp,
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_sparse import DeepseekV4CPUAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
swiglu_limit: float | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
is_sequence_parallel: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# is_sequence_parallel: weights are replicated, no collective ops
|
||||
# needed. Otherwise: standard TP with an allreduce at the end.
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
if swiglu_limit is not None:
|
||||
self.act_fn = SiluAndMulWithClamp(swiglu_limit)
|
||||
else:
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.prefix = prefix
|
||||
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.n_routed_experts = config.n_routed_experts
|
||||
self.n_activated_experts = config.num_experts_per_tok
|
||||
self.moe_intermediate_size = config.moe_intermediate_size
|
||||
self.swiglu_limit = config.swiglu_limit
|
||||
self.renormalize = config.norm_topk_prob
|
||||
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
|
||||
|
||||
self.gate = GateLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.n_routed_experts,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
|
||||
"hash_moe"
|
||||
)
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
# Use randint instead of empty to avoid garbage values causing
|
||||
# invalid memory access in dummy mode (--load-format="dummy")
|
||||
self.gate.tid2eid = nn.Parameter(
|
||||
torch.randint(
|
||||
0,
|
||||
config.n_routed_experts,
|
||||
(config.vocab_size, config.num_experts_per_tok),
|
||||
dtype=self.hash_indices_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
elif getattr(config, "topk_method", None) == "noaux_tc":
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
|
||||
# No CPU fused-shared-expert grouped-GEMM path yet.
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
|
||||
self.shared_experts = DeepseekV4MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
assert config.n_routed_experts % self.tp_size == 0
|
||||
|
||||
self.n_local_experts = config.n_routed_experts // self.tp_size
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
if self.gate.tid2eid is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
|
||||
class DeepseekV4DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Lazy import to avoid top-level tilelang dependency.
|
||||
# Registers both torch.ops.vllm.mhc_pre and mhc_post
|
||||
import vllm.model_executor.layers.mhc # noqa: F401
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.attn = DeepseekV4CPUAttention(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_post_alpha = 2.0
|
||||
mix_hc = (2 + self.hc_mult) * self.hc_mult
|
||||
hc_dim = self.hc_mult * self.hidden_size
|
||||
self.hc_attn_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
(mix_hc, hc_dim),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
(mix_hc, hc_dim),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_attn_base = nn.Parameter(
|
||||
torch.empty(
|
||||
mix_hc,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_base = nn.Parameter(
|
||||
torch.empty(
|
||||
mix_hc,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_attn_scale = nn.Parameter(
|
||||
torch.empty(
|
||||
3,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_scale = nn.Parameter(
|
||||
torch.empty(
|
||||
3,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.mhc_pre = MHCPreOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
self.mhc_fused_post_pre = MHCFusedPostPreOp()
|
||||
self.use_fused_mhc = HAS_TILELANG_MHC and not (
|
||||
HAS_AITER_MHC and self.hidden_size % 256 == 0
|
||||
)
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
):
|
||||
post_mix, res_mix, layer_input = self.mhc_pre(
|
||||
residual=x,
|
||||
fn=hc_fn,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
)
|
||||
return layer_input, post_mix, res_mix
|
||||
|
||||
def hc_post(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
):
|
||||
return self.mhc_post(x, residual, post, comb)
|
||||
|
||||
def _forward_fused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
# Run standalone hc_pre on first layer
|
||||
residual = x
|
||||
x, post_mix, res_mix = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
else:
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
def _forward_unfused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x, None, None, None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
if not self.use_fused_mhc:
|
||||
return self._forward_unfused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
return self._forward_fused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
}
|
||||
)
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
|
||||
self.device = current_platform.device_type
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV4MoE,
|
||||
"ffn",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
self.hc_mult,
|
||||
self.hc_dim,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(
|
||||
self.hc_mult,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_op = HCHeadOp()
|
||||
spec_config = vllm_config.speculative_config
|
||||
needs_mtp_hidden_states = spec_config is not None and (
|
||||
spec_config.use_eagle() or spec_config.uses_draft_model()
|
||||
)
|
||||
if get_pp_group().is_last_rank and needs_mtp_hidden_states:
|
||||
self._mtp_hidden_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.hc_dim,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
self._mtp_hidden_buffer = None
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> IntermediateTensors:
|
||||
# PP intermediate tensors keep the (num_tokens, hc_mult, hidden_size)
|
||||
# multi-stream shape until hc_head() collapses it.
|
||||
return IntermediateTensors(
|
||||
{
|
||||
"hidden_states": torch.zeros(
|
||||
(batch_size, self.hc_mult, self.config.hidden_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
hidden_states = hidden_states.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
# EAGLE3/DSpark/DFlash aux hidden states: post-mhc hidden state at
|
||||
# each target layer, averaged over hc_mult streams to [T,
|
||||
# hidden_size]. Empty unless a draft model set aux_hidden_state_layers.
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
# Reused below to avoid computing hc_post twice if the last layer is
|
||||
# also an aux layer.
|
||||
final_aux_recon: torch.Tensor | None = None
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
if (idx + 1) in self.aux_hidden_state_layers:
|
||||
# Unfused path already applied hc_post; fused path needs it
|
||||
# applied here before averaging.
|
||||
if layer.use_fused_mhc:
|
||||
aux_recon = layer.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
final_aux_recon = aux_recon
|
||||
else:
|
||||
aux_recon = hidden_states
|
||||
aux_hidden_states.append(aux_recon.mean(dim=1))
|
||||
if layer is not None and layer.use_fused_mhc:
|
||||
# Reuse the last layer's hc_post output if already computed above.
|
||||
if (
|
||||
final_aux_recon is not None
|
||||
and self.end_layer in self.aux_hidden_state_layers
|
||||
):
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = layer.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
self.hc_head_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
("compressor.fused_wkv_wgate", "compressor.wkv", 0),
|
||||
("compressor.fused_wkv_wgate", "compressor.wgate", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_param_name(name: str) -> str:
|
||||
inv_name = f"{name}_inv"
|
||||
if name not in params_dict and inv_name in params_dict:
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
# Each MoE's own per-layer fusion decision is the source of truth,
|
||||
# so the redirect below can't diverge from how it was built.
|
||||
fuse_by_layer = {
|
||||
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
|
||||
for mod_name, mod in self.named_modules()
|
||||
if isinstance(mod, DeepseekV4MoE)
|
||||
}
|
||||
n_routed = self.config.n_routed_experts
|
||||
# Maps the single shared-expert tensor group to one appended slot;
|
||||
# multiple shared experts would need per-expert slicing (see
|
||||
# deepseek_v2.py). DeepSeek-V4 has n_shared_experts == 1.
|
||||
if any(fuse_by_layer.values()) and self.config.n_shared_experts != 1:
|
||||
raise NotImplementedError(
|
||||
"deepseek-v4 fused shared-expert loading supports only "
|
||||
f"n_shared_experts == 1, got {self.config.n_shared_experts}"
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Shared-expert fusion: redirect shared_experts.w{1,2,3} into
|
||||
# the appended routed-expert slot so it loads through the
|
||||
# routed expert (grouped GEMM) loader.
|
||||
if ".ffn.shared_experts.w" in name and fuse_by_layer.get(
|
||||
extract_layer_index(name), False
|
||||
):
|
||||
name = name.replace(
|
||||
".ffn.shared_experts.w",
|
||||
f".ffn.experts.{n_routed}.w",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
break
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(param_name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# E8M0 scales are float8_e8m0fnu in checkpoints but the
|
||||
# MoE param is uint8; copy_() would numerically convert
|
||||
# (e.g. 2^-7 -> 0) instead of preserving raw bytes.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# return_success lets us skip experts that have
|
||||
# another replica available.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(param_name)
|
||||
continue
|
||||
|
||||
return loaded_params
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# (param_name, weight_name, expert_id, shard_id). When fusing shared
|
||||
# experts, include the appended slots so redirected shared-expert
|
||||
# weights route through the expert loader.
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.n_routed_experts + (
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
|
||||
def _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype: str, fuse_shared_experts: bool = False
|
||||
) -> WeightsMapper:
|
||||
if expert_dtype == "fp4":
|
||||
# MXFP4 experts register scales as ``w{1,2,3}_weight_scale`` (no
|
||||
# _inv); FP8 linear/shared-expert scales register as
|
||||
# ``weight_scale_inv``. Route DeepSeek-native ``.scale`` and
|
||||
# AMD-Quark ``.weight_scale`` keys accordingly, leaving expert/
|
||||
# shared-expert ``w{1,2,3}.weight_scale`` untouched for the MXFP4
|
||||
# loader.
|
||||
scale_regex = {
|
||||
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
|
||||
}
|
||||
else:
|
||||
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
|
||||
# scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys
|
||||
# there.
|
||||
scale_regex = {
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
}
|
||||
# Fused shared experts are redirected to routed expert slots elsewhere;
|
||||
# leave their names untouched here.
|
||||
substr_map: dict[str, str | None] = (
|
||||
{}
|
||||
if fuse_shared_experts
|
||||
else {".shared_experts.w2": ".shared_experts.down_proj"}
|
||||
)
|
||||
# No MTP support on CPU yet: drop mtp.* weights (no model.mtp submodule).
|
||||
substr_map["mtp."] = None
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"layers.": "model.layers.",
|
||||
"embed.": "model.embed.",
|
||||
"norm.": "model.norm.",
|
||||
"hc_head": "model.hc_head",
|
||||
"mtp.": "model.mtp.",
|
||||
},
|
||||
orig_to_new_regex=scale_regex,
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
".input_scale": ".input_scale_2",
|
||||
},
|
||||
orig_to_new_substr=substr_map,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
model_cls = DeepseekV4Model
|
||||
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype,
|
||||
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# No CPU AMX weight-packing hook yet.
|
||||
pass
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
@@ -1,19 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 MTP (speculative decoding) — deferred, not implemented.
|
||||
|
||||
``DeepSeekV4MTP`` is only instantiated when
|
||||
``vllm/model_executor/models/registry.py``'s speculative-decoding registry
|
||||
resolves it by name for a speculative-decoding config; a plain (non-spec)
|
||||
DeepSeek-V4 CPU model never constructs this class. This stub exists solely
|
||||
so ``vllm.models.deepseek_v4.cpu`` is import-clean.
|
||||
"""
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class DeepSeekV4MTP:
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 speculative decoding (MTP) is not implemented on CPU."
|
||||
)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,600 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark draft model for DeepSeek-V4 (semi-autoregressive speculative decoding).
|
||||
|
||||
See: qwen3_dspark.py for base architecture. This one is specialized to the DSV4 DSpark,
|
||||
which reuses the target model's architecture similarly to MTP.
|
||||
|
||||
To implement non-causal attention, we leverage the sparse attention implementation to
|
||||
include the future query tokens in the top-k indices for each query token.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
DeepseekV4Model,
|
||||
DeepseekV4MoE,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
|
||||
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$")
|
||||
|
||||
|
||||
def _duplicate_context_wkv_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Load every draft layer's WKV into the cross-layer projection."""
|
||||
for name, weight in weights:
|
||||
yield name, weight
|
||||
match = _CONTEXT_WKV_RE.fullmatch(name)
|
||||
if match is None:
|
||||
continue
|
||||
layer_idx = int(match.group(1))
|
||||
if layer_idx >= num_layers:
|
||||
continue
|
||||
stacked_weight = weight.detach()
|
||||
stacked_weight.shard_id = layer_idx
|
||||
yield f"context_wkv_proj.{match.group(2)}", stacked_weight
|
||||
|
||||
|
||||
class DSparkDeepseekV4Model(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_eps = config.hc_eps
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
|
||||
|
||||
# Shared with the target (aliased by the speculator's loading utility).
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
self.main_proj = ReplicatedLinear(
|
||||
config.hidden_size * len(self.target_layer_ids),
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "main_proj"),
|
||||
)
|
||||
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4DecoderLayer(
|
||||
current_vllm_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
)
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
)
|
||||
self.context_wkv_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
[config.head_dim] * self.num_dspark_layers,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "context_wkv_proj"),
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov + confidence heads
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", True):
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
config.hidden_size + config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
|
||||
|
||||
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
|
||||
"""
|
||||
return self.main_norm(self.main_proj(aux_hidden_states))
|
||||
|
||||
@torch.inference_mode()
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
main_x: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Insert the sliding-window context KV for every draft layer.
|
||||
|
||||
Mirrors the reference DSparkAttention: each layer derives its context KV
|
||||
from the SAME projected target hidden ``main_x``, via that layer's own
|
||||
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
|
||||
layer's context slots.
|
||||
|
||||
``context_slot_mappings`` is a per-layer list (each entry is the context
|
||||
slot mapping for that layer's kv-cache group, since the hybrid manager may
|
||||
place draft layers in different groups). ``None`` (or a ``None`` entry)
|
||||
runs the projection to reserve workspace but writes nothing (profiling).
|
||||
"""
|
||||
all_kv = self.context_wkv_proj(main_x).view(
|
||||
main_x.shape[0], self.num_dspark_layers, self.config.head_dim
|
||||
)
|
||||
for i, (layer, kv) in enumerate(
|
||||
zip(self.layers, all_kv.unbind(1), strict=True)
|
||||
):
|
||||
slot_mapping = (
|
||||
None if context_slot_mappings is None else context_slot_mappings[i]
|
||||
)
|
||||
attn = layer.attn
|
||||
kv = attn.kv_norm(kv)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
_insert_context_kv(attn, kv, context_positions, slot_mapping)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, inputs_embeds
|
||||
)
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
input_ids = sp_shard(input_ids)
|
||||
mega_gate_metadata = None
|
||||
if self.use_native_mega_moe:
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
input_ids,
|
||||
has_hash_routing=False,
|
||||
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
|
||||
if getattr(self.config, "vision_n_layers", 0) > 0
|
||||
else None,
|
||||
)
|
||||
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
|
||||
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
|
||||
residual = post_mix = res_mix = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
mega_gate_metadata=mega_gate_metadata,
|
||||
)
|
||||
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
# hc_head reduces the hc copies; return the PRE-norm head hidden
|
||||
hidden_states = hc_head_fused_kernel_tilelang(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
self.hc_head_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _insert_context_kv(
|
||||
attn: nn.Module,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
|
||||
|
||||
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
|
||||
query and discard it, since context tokens have no query). Mirrors
|
||||
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
|
||||
"""
|
||||
swa_cache = attn.swa_cache_layer.kv_cache
|
||||
block_size = attn.swa_cache_layer.block_size
|
||||
cos_sin_cache = attn.rotary_emb.cos_sin_cache
|
||||
cache_dtype = swa_cache.dtype
|
||||
n_ctx = kv.shape[0]
|
||||
dummy_q = torch.zeros(
|
||||
(n_ctx, attn.n_local_heads, attn.head_dim),
|
||||
dtype=kv.dtype,
|
||||
device=kv.device,
|
||||
)
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged layout
|
||||
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.padded_heads,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
elif cache_dtype == torch.bfloat16:
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
else: # per-tensor fp8 (torch.float8_e4m3fn)
|
||||
# TODO(ben): double-check if this is being dispatched correctly for FI backend
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
dummy_q_fp8,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn._flashinfer_fp8_kv_scale,
|
||||
attn._flashinfer_fp8_q_scale_inv,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
has_own_lm_head = False
|
||||
# Full-vocab draft: draft ids are target ids, no remapping needed.
|
||||
draft_id_to_target_id = None
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.pad_shared_expert = getattr(
|
||||
self.quant_config, "weight_block_size", None
|
||||
) is not None and not _use_sequence_parallel(vllm_config)
|
||||
self.model = DSparkDeepseekV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
# Shared with the target (aliased by the speculator's load utility).
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.combine_hidden_states(aux_hidden_states)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
|
||||
# layer, named by its prefix.
|
||||
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
|
||||
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
self.model.precompute_and_store_context_kv(
|
||||
context_states, context_positions, context_slot_mappings
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Returns the pre-norm hc_head hidden ([T, hidden_size]).
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Base logits U_k = lm_head(norm(head_hidden))."""
|
||||
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Full-vocab draft: base logits, no d2t scatter.
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids # full-vocab: draft ids are target ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
# --- Weight loading ----------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
|
||||
|
||||
Non-mtp weights (embed/head/main layers) belong to the target model and
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
first_layer = self.model.layers[0]
|
||||
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
|
||||
if use_native_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
else:
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=first_layer.ffn.n_redundant_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
if "confidence_head." in name:
|
||||
loaded_confidence_head = True
|
||||
|
||||
# ``.scale`` -> per-method scale suffix.
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts." in name:
|
||||
experts = self.model.layers[extract_layer_index(name)].ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
|
||||
self.quant_config, name, loaded_weight
|
||||
)
|
||||
|
||||
if name.startswith("model.context_wkv_proj."):
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, loaded_weight.shard_id)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
# E8M0 expert scales: keep raw exponent bytes.
|
||||
if ".experts." in name:
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
success = param.weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
|
||||
# Stacked rules only apply to decoder-layer weights. Head-stack params
|
||||
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
|
||||
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
Returns None for non-mtp weights (owned by the target model).
|
||||
"""
|
||||
if name.startswith("context_wkv_proj."):
|
||||
return f"model.{name}"
|
||||
m = re.match(r"mtp\.(\d+)\.(.*)", name)
|
||||
if m is None:
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
|
||||
return None
|
||||
# Head-stack params live at model level (mtp.last), context combiner at
|
||||
# model level (mtp.0); everything else is a per-layer decoder block.
|
||||
head_prefixes = (
|
||||
"norm.",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
"markov_head.",
|
||||
"confidence_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
):
|
||||
return f"model.{rest}"
|
||||
return f"model.layers.{stage}.{rest}"
|
||||
@@ -1,986 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4 FlashInfer sparse MLA backend."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
rope_quant_attn_out,
|
||||
rope_quant_unsupported_reason,
|
||||
)
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWABackend,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
_flashinfer_dsv4_workspace_by_device: dict[torch.device, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
workspace = _flashinfer_dsv4_workspace_by_device.get(device)
|
||||
if workspace is None:
|
||||
workspace = torch.zeros(
|
||||
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
_flashinfer_dsv4_workspace_by_device[device] = workspace
|
||||
return workspace
|
||||
|
||||
|
||||
def _packed_block_span(pool: torch.Tensor) -> int:
|
||||
"""Per-block stride of ``pool`` in tokens (``stride(0)//stride(-2)``): ==
|
||||
block_size for unpacked KV, larger when packed (#44577). Raises if not
|
||||
token-aligned."""
|
||||
block_stride = pool.stride(0)
|
||||
token_stride = pool.stride(-2)
|
||||
if block_stride % token_stride != 0:
|
||||
raise NotImplementedError(
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 packed KV requires the per-block stride "
|
||||
f"({block_stride}) to be a multiple of the per-token stride "
|
||||
f"({token_stride}); this layout is not supported yet."
|
||||
)
|
||||
return block_stride // token_stride
|
||||
|
||||
|
||||
# Sparse MLA h_q counts accepted natively (flashinfer>=0.6.14, #3545).
|
||||
_SPARSE_MLA_SUPPORTED_Q_HEADS = (8, 16, 32, 64, 128)
|
||||
|
||||
|
||||
def _pad_to_supported_q_heads(num_heads: int) -> int:
|
||||
for supported in _SPARSE_MLA_SUPPORTED_Q_HEADS:
|
||||
if num_heads <= supported:
|
||||
return supported
|
||||
raise ValueError(
|
||||
f"DeepseekV4 FlashInfer MLA Sparse does not support {num_heads} heads "
|
||||
"(sparse MLA kernel requires h_q in {8, 16, 32, 64, 128})."
|
||||
)
|
||||
|
||||
|
||||
def _required_sm120_sparse_topk(vllm_config: VllmConfig, window_size: int) -> int:
|
||||
"""Return the SM120 DSV4 SWA specialization needed by this model."""
|
||||
if not vllm_config.attention_config.use_non_causal:
|
||||
return window_size
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None:
|
||||
return window_size
|
||||
return get_dspark_swa_index_width(
|
||||
window_size,
|
||||
speculative_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
|
||||
|
||||
Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder``
|
||||
"""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
"fp8_ds_mla",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4"
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [512]
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major in [10, 12]
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
if device_capability.major == 10:
|
||||
if kv_cache_dtype == "fp8_ds_mla":
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 SM10x uses the plain "
|
||||
"per-tensor FP8 KV layout, not fp8_ds_mla"
|
||||
)
|
||||
if kv_cache_dtype not in (None, "auto", "bfloat16", "fp8", "fp8_e4m3"):
|
||||
return "kv_cache_dtype not supported"
|
||||
return None
|
||||
if device_capability.major == 12:
|
||||
if kv_cache_dtype not in ("fp8", "fp8_e4m3", "fp8_ds_mla"):
|
||||
return "kv_cache_dtype not supported"
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
|
||||
|
||||
if not has_flashinfer_sparse_mla_sm120():
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 SM120 requires FlashInfer's "
|
||||
"sparse MLA decode API"
|
||||
)
|
||||
return None
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4 requires SM10x or SM12x"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4FlashInferSparseMLAMetadataBuilder"]:
|
||||
return DeepseekV4FlashInferSparseMLAMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4FlashInferSparseMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
"""Varlen-capable metadata builder for the FlashInfer sparse MLA backend."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
"""SWA metadata for the FlashInfer sparse decode path (varlen decode)."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekSparseSWAFlashInferMetadataBuilder]:
|
||||
return DeepseekSparseSWAFlashInferMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
"""FlashInfer TRTLLM-gen sparse MLA attention layer for SM100 DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4FlashInferMLASparseBackend
|
||||
swa_backend_cls = DeepseekSparseSWAFlashInferBackend
|
||||
use_fp8_ds_mla_layout: ClassVar[bool] = False
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(
|
||||
self, o: torch.Tensor | QuantizedActivation, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self.wo_b,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
|
||||
def _alloc_attn_out(
|
||||
self, num_tokens: int, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor | QuantizedActivation:
|
||||
if self._fuse_rope_quant:
|
||||
return rope_quant_attn_out(num_tokens, hidden_states.device)
|
||||
return super()._alloc_attn_out(num_tokens, hidden_states)
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
|
||||
# RopeQuant derives each query's RoPE position as seq_len - q_len + i,
|
||||
# which matches `positions` for DSpark's non-causal draft block too,
|
||||
# since every call passes the real per-request seq_lens. The one
|
||||
# exception is a draft block running past max_model_len, whose clamped
|
||||
# positions it does not reproduce; that only costs those drafts.
|
||||
reason = rope_quant_unsupported_reason(self)
|
||||
self._fuse_rope_quant = reason is None
|
||||
if reason is None:
|
||||
logger.info_once(
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 fuses the inverse RoPE and FP8 "
|
||||
"quant of the attention output."
|
||||
)
|
||||
else:
|
||||
logger.debug_once("FLASHINFER_MLA_SPARSE_DSV4 RopeQuant off: %s", reason)
|
||||
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
|
||||
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
|
||||
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
fp8_q_scale = 1.0
|
||||
fp8_kv_scale = 1.0
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale",
|
||||
torch.tensor([fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale_inv",
|
||||
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_kv_scale",
|
||||
torch.tensor([fp8_kv_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
# TRTLLM-gen takes scalar scale args on a distinct C++ path vs
|
||||
# one-element tensors, so these are Python floats.
|
||||
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
|
||||
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor | QuantizedActivation,
|
||||
) -> None:
|
||||
if isinstance(output, QuantizedActivation):
|
||||
assert output.data.shape[0] == q.shape[0]
|
||||
else:
|
||||
# The TRTLLM-gen kernel requires h_q in {64, 128}, so the output
|
||||
# buffer is allocated at the padded head count while q arrives at
|
||||
# the local head count; _forward pads q to match before the launcher.
|
||||
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
|
||||
f"output buffer shape {output.shape} incompatible with q {q.shape}"
|
||||
)
|
||||
assert output.shape[1] >= q.shape[1], (
|
||||
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
|
||||
)
|
||||
# Per-tensor FP8 q produces a bf16 attention output.
|
||||
expected_output_dtype = (
|
||||
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
|
||||
)
|
||||
assert output.dtype == expected_output_dtype, (
|
||||
f"output dtype {output.dtype} must match expected "
|
||||
f"{expected_output_dtype} for q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: FlashInfer reads the cache directly and lazily
|
||||
# allocates its workspace, so nothing to reserve here.
|
||||
if isinstance(output, QuantizedActivation):
|
||||
output.data.zero_()
|
||||
output.scale.zero_()
|
||||
else:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
# SWA-only layers don't allocate their own compressed KV cache.
|
||||
self_kv_cache = self.kv_cache if not swa_only else None
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
|
||||
self._forward(
|
||||
q=q,
|
||||
kv_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output,
|
||||
)
|
||||
|
||||
def _build_sparse_index_metadata(
|
||||
self,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
num_rows: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build the combined sparse-index tensors for the mixed batch.
|
||||
|
||||
Returns ``(compressed_kv_cache, seq_lens, sparse_indices,
|
||||
sparse_topk_lens)``, sized ``num_rows`` (default: the real tokens).
|
||||
"""
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_reqs = num_decodes + num_prefills
|
||||
num_tokens = num_decode_tokens + num_prefill_tokens
|
||||
|
||||
assert swa_metadata.seq_lens is not None
|
||||
assert swa_metadata.query_start_loc is not None
|
||||
assert swa_metadata.token_to_req_indices is not None
|
||||
assert swa_metadata.decode_swa_indices is not None
|
||||
assert swa_metadata.block_table is not None
|
||||
|
||||
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
|
||||
num_decode_tokens, swa_metadata.decode_swa_width
|
||||
)
|
||||
decode_compressed_topk_lens = None
|
||||
decode_compressed_indices_are_local = False
|
||||
decode_is_valid_token = None
|
||||
|
||||
if swa_only:
|
||||
assert self.topk_indices_buffer is not None
|
||||
compressed_kv_cache = swa_k_cache
|
||||
decode_compressed_indices = None
|
||||
prefill_topk_indices = self.topk_indices_buffer[
|
||||
num_decode_tokens:num_tokens, :0
|
||||
]
|
||||
compressed_block_table = None
|
||||
compressed_block_size = swa_metadata.block_size
|
||||
top_k = 0
|
||||
else:
|
||||
assert kv_cache is not None
|
||||
assert attn_metadata is not None
|
||||
compressed_kv_cache = kv_cache
|
||||
compressed_block_table = attn_metadata.block_table[:num_reqs]
|
||||
compressed_block_size = attn_metadata.block_size // self.compress_ratio
|
||||
|
||||
if self.compress_ratio == 4:
|
||||
assert self.topk_indices_buffer is not None
|
||||
if num_prefill_tokens > 0:
|
||||
prefill_topk_indices = self.topk_indices_buffer[
|
||||
num_decode_tokens:num_tokens
|
||||
]
|
||||
top_k = prefill_topk_indices.shape[-1]
|
||||
else:
|
||||
prefill_topk_indices = self.topk_indices_buffer[:0, :0]
|
||||
top_k = 0
|
||||
|
||||
decode_compressed_indices_are_local = True
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
decode_is_valid_token = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if num_decode_tokens > 0:
|
||||
decode_compressed_indices = self.topk_indices_buffer[
|
||||
:num_decode_tokens
|
||||
]
|
||||
else:
|
||||
# Keep the logical width aligned with the mixed-batch case so
|
||||
# pure-prefill steps reuse the same Triton specialization.
|
||||
decode_compressed_indices = prefill_topk_indices[:0]
|
||||
else:
|
||||
if num_prefill_tokens > 0:
|
||||
assert attn_metadata.c128a_prefill_topk_indices is not None
|
||||
prefill_topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
top_k = prefill_topk_indices.shape[-1]
|
||||
else:
|
||||
prefill_topk_indices = decode_swa_indices[:0, :0]
|
||||
top_k = 0
|
||||
|
||||
if num_decode_tokens > 0:
|
||||
assert attn_metadata.c128a_global_decode_topk_indices is not None
|
||||
assert attn_metadata.c128a_decode_topk_lens is not None
|
||||
decode_compressed_indices = (
|
||||
attn_metadata.c128a_global_decode_topk_indices.view(
|
||||
num_decode_tokens, -1
|
||||
)
|
||||
)
|
||||
decode_compressed_topk_lens = attn_metadata.c128a_decode_topk_lens
|
||||
if num_prefill_tokens == 0:
|
||||
prefill_topk_indices = decode_compressed_indices[:0, :0]
|
||||
else:
|
||||
decode_compressed_indices = prefill_topk_indices[:0]
|
||||
decode_compressed_topk_lens = swa_metadata.seq_lens[:0]
|
||||
|
||||
query_start_loc = swa_metadata.query_start_loc[: num_reqs + 1]
|
||||
seq_lens = swa_metadata.seq_lens[:num_reqs]
|
||||
assert seq_lens.dtype == torch.int32
|
||||
# cache for SWA-only and C128A that build the same mixed sparse indices
|
||||
# C4A stays uncached.
|
||||
cache_key = (
|
||||
"swa_only"
|
||||
if swa_only
|
||||
else ("c128a" if self.compress_ratio == 128 else "c4a")
|
||||
)
|
||||
if num_rows is not None:
|
||||
cache_key = f"{cache_key}:{num_rows}"
|
||||
cached_sparse = swa_metadata.flashinfer_sparse_index_cache.get(cache_key, None)
|
||||
if cached_sparse is None:
|
||||
swa_block_span = _packed_block_span(swa_k_cache)
|
||||
compressed_block_span = _packed_block_span(compressed_kv_cache)
|
||||
sparse_indices, sparse_topk_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices,
|
||||
decode_compressed_indices,
|
||||
decode_compressed_topk_lens,
|
||||
prefill_topk_indices[:num_prefill_tokens],
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
swa_metadata.token_to_req_indices[:num_tokens],
|
||||
swa_metadata.block_table[:num_reqs],
|
||||
swa_metadata.block_size,
|
||||
compressed_block_table,
|
||||
compressed_block_size,
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
decode_compressed_indices_are_local=decode_compressed_indices_are_local,
|
||||
decode_is_valid_token=decode_is_valid_token,
|
||||
swa_block_span=swa_block_span,
|
||||
compressed_block_span=compressed_block_span,
|
||||
prefill_left_visible=swa_metadata.prefill_left_visible,
|
||||
prefill_right_visible=swa_metadata.prefill_right_visible,
|
||||
# getattr for tests that bypass __init__ via object.__new__.
|
||||
max_image_tokens=getattr(self, "max_image_tokens", 0),
|
||||
num_rows=num_rows,
|
||||
)
|
||||
if not cache_key.startswith("c4a"):
|
||||
swa_metadata.flashinfer_sparse_index_cache[cache_key] = (
|
||||
sparse_indices,
|
||||
sparse_topk_lens,
|
||||
)
|
||||
else:
|
||||
sparse_indices, sparse_topk_lens = cached_sparse
|
||||
return compressed_kv_cache, seq_lens, sparse_indices, sparse_topk_lens
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor | QuantizedActivation,
|
||||
) -> None:
|
||||
assert self.kv_cache_torch_dtype in (torch.bfloat16, torch.float8_e4m3fn)
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_reqs = num_decodes + num_prefills
|
||||
num_tokens = num_decode_tokens + num_prefill_tokens
|
||||
if num_tokens == 0:
|
||||
return
|
||||
# RopeQuant's output is group-major with a group stride of exactly
|
||||
# sum_q tokens, so both calls span every row of the (CUDA-graph
|
||||
# padded) buffers and select their tokens through cum_seq_lens_q
|
||||
# alone; the kernel never touches rows outside it.
|
||||
rope_quant = isinstance(output, QuantizedActivation)
|
||||
|
||||
(
|
||||
compressed_kv_cache,
|
||||
seq_lens,
|
||||
sparse_indices,
|
||||
sparse_topk_lens,
|
||||
) = self._build_sparse_index_metadata(
|
||||
kv_cache=kv_cache,
|
||||
swa_k_cache=swa_k_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=attn_metadata,
|
||||
swa_only=swa_only,
|
||||
num_rows=q.shape[0] if rope_quant else None,
|
||||
)
|
||||
|
||||
# CUDA graph execution can pad q/output past the scheduled token count;
|
||||
# restrict to the real tokens (the launcher validates sparse indices).
|
||||
query = q if rope_quant else q[:num_tokens]
|
||||
bmm1_scale: float | torch.Tensor = self.scale
|
||||
bmm2_scale: float | torch.Tensor = 1.0
|
||||
if self.kv_cache_torch_dtype == torch.float8_e4m3fn:
|
||||
assert query.dtype == torch.float8_e4m3fn
|
||||
bmm1_scale = self._flashinfer_fp8_bmm1_scale
|
||||
bmm2_scale = self._flashinfer_fp8_bmm2_scale
|
||||
else:
|
||||
assert query.dtype == torch.bfloat16
|
||||
query = query.contiguous()
|
||||
|
||||
# The TRTLLM-gen sparse-MLA kernel requires h_q in {64, 128}; zero-pad
|
||||
# the query heads to the allocated output head count. Padded heads attend
|
||||
# to the shared KV and are sliced off downstream (output is padded too).
|
||||
padded_heads = (
|
||||
output.shape[1] if isinstance(output, torch.Tensor) else self.padded_heads
|
||||
)
|
||||
if query.shape[1] < padded_heads:
|
||||
padded_query = query.new_zeros(
|
||||
(query.shape[0], padded_heads, query.shape[2])
|
||||
)
|
||||
padded_query[:, : query.shape[1], :] = query
|
||||
query = padded_query
|
||||
|
||||
workspace = _get_flashinfer_dsv4_workspace(q.device)
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
assert query_start_loc is not None and query_start_loc_cpu is not None
|
||||
|
||||
def run(
|
||||
rows: slice, reqs: slice, cum_seq_lens_q: torch.Tensor, max_q_len: int
|
||||
) -> None:
|
||||
if rope_quant:
|
||||
assert isinstance(output, QuantizedActivation)
|
||||
rows = slice(None)
|
||||
out_kwargs = dict(
|
||||
out=output.data,
|
||||
dsv4_output_scale=output.scale,
|
||||
dsv4_inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache,
|
||||
)
|
||||
else:
|
||||
assert isinstance(output, torch.Tensor)
|
||||
out_kwargs = dict(out=output[rows])
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=query[rows],
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[rows],
|
||||
sparse_indices_are_storage_offsets=True,
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[rows],
|
||||
seq_lens=seq_lens[reqs],
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=self.attn_sink,
|
||||
cum_seq_lens_q=cum_seq_lens_q,
|
||||
max_q_len=max_q_len,
|
||||
**out_kwargs,
|
||||
)
|
||||
|
||||
# Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for
|
||||
# uniform-q batches, and this avoids flattening mixed batches into one call.
|
||||
if num_decode_tokens > 0:
|
||||
run(
|
||||
slice(0, num_decode_tokens),
|
||||
slice(0, num_decodes),
|
||||
query_start_loc[: num_decodes + 1],
|
||||
swa_metadata.max_decode_query_len,
|
||||
)
|
||||
|
||||
if num_prefill_tokens > 0:
|
||||
prefill_cu = query_start_loc[num_decodes : num_reqs + 1]
|
||||
if not rope_quant:
|
||||
# The prefill query view re-anchors at offset 0, so rebase the
|
||||
# cumulative query offsets to start at 0.
|
||||
prefill_cu = prefill_cu - query_start_loc[num_decodes]
|
||||
prefill_cu_cpu = query_start_loc_cpu[num_decodes : num_reqs + 1]
|
||||
prefill_lens_cpu = prefill_cu_cpu[1:] - prefill_cu_cpu[:-1]
|
||||
run(
|
||||
slice(num_decode_tokens, num_tokens),
|
||||
slice(num_decodes, num_reqs),
|
||||
prefill_cu,
|
||||
int(prefill_lens_cpu.max().item()),
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
"""DeepSeek V4 sparse MLA attention through FlashInfer's SM120 kernels."""
|
||||
|
||||
backend_cls = DeepseekV4FlashInferMLASparseBackend
|
||||
swa_backend_cls = DeepseekSparseSWAFlashInferBackend
|
||||
use_fp8_ds_mla_layout: ClassVar[bool] = True
|
||||
|
||||
@staticmethod
|
||||
def _get_workspace(device: torch.device) -> torch.Tensor:
|
||||
return _get_flashinfer_dsv4_workspace(device)
|
||||
|
||||
@staticmethod
|
||||
def _as_sparse_cache(kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
if kv_cache.dtype == torch.float8_e4m3fn:
|
||||
kv_cache = kv_cache.view(torch.uint8)
|
||||
if kv_cache.dim() == 4:
|
||||
return kv_cache
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self.wo_b,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120_config
|
||||
|
||||
required_topk = _required_sm120_sparse_topk(vllm_config, self.window_size)
|
||||
if not has_flashinfer_sparse_mla_sm120_config(self.padded_heads, required_topk):
|
||||
raise RuntimeError(
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires a FlashInfer "
|
||||
"DSV4 sparse MLA decode specialization for "
|
||||
f"(num_q_heads={self.padded_heads}, top_k={required_topk}). "
|
||||
"Install a FlashInfer build containing "
|
||||
"flashinfer-ai/flashinfer#4380."
|
||||
)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
|
||||
# Per-tensor FP8 cache path scales.
|
||||
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
fp8_q_scale = 1.0
|
||||
fp8_kv_scale = 1.0
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale",
|
||||
torch.tensor([fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale_inv",
|
||||
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_kv_scale",
|
||||
torch.tensor([fp8_kv_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
# FlashInfer expects scalar scale arguments for this path.
|
||||
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
|
||||
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
|
||||
|
||||
def _reserve_empty_forward_workspace(self) -> None:
|
||||
self._get_workspace(
|
||||
torch.device("cuda", torch.accelerator.current_device_index())
|
||||
)
|
||||
|
||||
def _forward_sparse_impl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
self_kv_cache: torch.Tensor | None,
|
||||
swa_kv_cache: torch.Tensor,
|
||||
swa_only: bool,
|
||||
) -> None:
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
if swa_metadata.num_prefills > 0:
|
||||
self._forward_prefill(
|
||||
q=q[num_decode_tokens:],
|
||||
compressed_k_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
output=output[num_decode_tokens:],
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
)
|
||||
if swa_metadata.num_decodes > 0:
|
||||
self._forward_decode(
|
||||
q=q[:num_decode_tokens],
|
||||
kv_cache=self_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
# Output may be padded to backend-supported head counts.
|
||||
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
|
||||
f"output buffer shape {output.shape} incompatible with q shape {q.shape}"
|
||||
)
|
||||
assert output.shape[1] >= q.shape[1], (
|
||||
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
|
||||
)
|
||||
# Per-tensor FP8 q produces a bf16 attention output.
|
||||
expected_output_dtype = (
|
||||
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
|
||||
)
|
||||
assert output.dtype == expected_output_dtype, (
|
||||
f"output dtype {output.dtype} must match expected {expected_output_dtype} "
|
||||
f"for q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
self._reserve_empty_forward_workspace()
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
# SWA-only layers don't allocate their own compressed KV cache.
|
||||
self_kv_cache = self.kv_cache if not swa_only else None
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
|
||||
self._forward_sparse_impl(
|
||||
q=q,
|
||||
output=output,
|
||||
flashmla_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
self_kv_cache=self_kv_cache,
|
||||
swa_kv_cache=swa_kv_cache,
|
||||
swa_only=swa_only,
|
||||
)
|
||||
|
||||
def _prepare_query(self, q: torch.Tensor, output: torch.Tensor) -> torch.Tensor:
|
||||
if self.kv_cache_torch_dtype == torch.float8_e4m3fn:
|
||||
assert q.dtype == torch.float8_e4m3fn
|
||||
q = q.to(torch.bfloat16)
|
||||
else:
|
||||
assert q.dtype == torch.bfloat16
|
||||
padded_heads = output.shape[1]
|
||||
if q.shape[1] < padded_heads:
|
||||
padded_query = q.new_zeros((q.shape[0], padded_heads, q.shape[2]))
|
||||
padded_query[:, : q.shape[1], :] = q
|
||||
q = padded_query
|
||||
return q.contiguous()
|
||||
|
||||
def _forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
extra_sparse_indices = None
|
||||
extra_sparse_lengths = None
|
||||
if not swa_only:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError(
|
||||
"Sparse MLA metadata is required for compressed layers."
|
||||
)
|
||||
if swa_metadata.is_valid_token is None:
|
||||
raise RuntimeError(
|
||||
"SWA validity metadata is required for compressed layers."
|
||||
)
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if self.compress_ratio == 4:
|
||||
if self.topk_indices_buffer is None:
|
||||
raise RuntimeError(
|
||||
"C4A decode requires top-k indices from the indexer."
|
||||
)
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
global_indices, extra_sparse_lengths = (
|
||||
compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
)
|
||||
)
|
||||
extra_sparse_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
else:
|
||||
extra_sparse_indices = attn_metadata.c128a_global_decode_topk_indices
|
||||
extra_sparse_lengths = attn_metadata.c128a_decode_topk_lens
|
||||
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
assert swa_indices is not None
|
||||
assert swa_lens is not None
|
||||
q = self._prepare_query(q, output)
|
||||
swa_cache = self._as_sparse_cache(self.swa_cache_layer.kv_cache)
|
||||
extra_cache = self._as_sparse_cache(kv_cache) if kv_cache is not None else None
|
||||
if extra_cache is not None and extra_sparse_indices is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA decode requires compressed sparse indices."
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=q,
|
||||
swa_kv_cache=swa_cache,
|
||||
workspace_buffer=self._get_workspace(q.device),
|
||||
sparse_indices=swa_indices,
|
||||
compressed_kv_cache=extra_cache,
|
||||
out=output,
|
||||
bmm1_scale=self.scale,
|
||||
sinks=self.attn_sink,
|
||||
kv_layout="NHD",
|
||||
swa_topk_lens=swa_lens,
|
||||
extra_sparse_indices=extra_sparse_indices,
|
||||
extra_sparse_topk_lens=extra_sparse_lengths,
|
||||
)
|
||||
|
||||
def _forward_prefill(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
compressed_k_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> None:
|
||||
swa_only = self.compress_ratio <= 1
|
||||
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
assert query_start_loc_cpu is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
|
||||
local_topk_indices: torch.Tensor | None
|
||||
if swa_only:
|
||||
local_topk_indices = None
|
||||
elif self.compress_ratio == 4:
|
||||
if self.topk_indices_buffer is None:
|
||||
raise RuntimeError(
|
||||
"C4A prefill requires top-k indices from the indexer."
|
||||
)
|
||||
local_topk_indices = self.topk_indices_buffer[
|
||||
num_decode_tokens : num_decode_tokens + num_prefill_tokens
|
||||
]
|
||||
else:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError("C128A prefill metadata is missing.")
|
||||
local_topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
|
||||
extra_sparse_indices: torch.Tensor | None = None
|
||||
extra_sparse_lengths: torch.Tensor | None = None
|
||||
if local_topk_indices is not None:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError("C4A prefill metadata is missing.")
|
||||
if swa_metadata.token_to_req_indices is None:
|
||||
raise RuntimeError("C4A prefill request mapping is missing.")
|
||||
if swa_metadata.is_valid_token is None:
|
||||
raise RuntimeError("C4A prefill validity metadata is missing.")
|
||||
prefill_token_slice = slice(
|
||||
num_decode_tokens, num_decode_tokens + num_prefill_tokens
|
||||
)
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
extra_sparse_indices, extra_sparse_lengths = (
|
||||
compute_global_topk_indices_and_lens(
|
||||
local_topk_indices,
|
||||
swa_metadata.token_to_req_indices[prefill_token_slice],
|
||||
attn_metadata.block_table,
|
||||
block_size,
|
||||
swa_metadata.is_valid_token[prefill_token_slice],
|
||||
)
|
||||
)
|
||||
|
||||
assert swa_metadata.prefill_swa_indices is not None
|
||||
assert swa_metadata.prefill_swa_lens is not None
|
||||
|
||||
q = self._prepare_query(q, output)
|
||||
swa_kv_paged = self._as_sparse_cache(swa_k_cache)
|
||||
if swa_only:
|
||||
extra_kv_paged = None
|
||||
else:
|
||||
if compressed_k_cache is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA layers require their compressed KV cache."
|
||||
)
|
||||
extra_kv_paged = self._as_sparse_cache(compressed_k_cache)
|
||||
|
||||
num_chunks = (
|
||||
num_prefills + self.PREFILL_CHUNK_SIZE - 1
|
||||
) // self.PREFILL_CHUNK_SIZE
|
||||
for chunk_idx in range(num_chunks):
|
||||
chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE
|
||||
chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills)
|
||||
query_start = (
|
||||
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
||||
)
|
||||
query_end = (
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
|
||||
extra_sparse_indices_chunk = (
|
||||
extra_sparse_indices[query_start:query_end]
|
||||
if extra_sparse_indices is not None
|
||||
else None
|
||||
)
|
||||
extra_sparse_lengths_chunk = (
|
||||
extra_sparse_lengths[query_start:query_end]
|
||||
if extra_sparse_lengths is not None
|
||||
else None
|
||||
)
|
||||
|
||||
q_chunk = q[query_start:query_end]
|
||||
swa_indices_chunk = swa_metadata.prefill_swa_indices[query_start:query_end]
|
||||
swa_lens_chunk = swa_metadata.prefill_swa_lens[query_start:query_end]
|
||||
if extra_kv_paged is not None and extra_sparse_indices_chunk is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA prefill requires compressed sparse indices."
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=q_chunk,
|
||||
swa_kv_cache=swa_kv_paged,
|
||||
workspace_buffer=self._get_workspace(q.device),
|
||||
sparse_indices=swa_indices_chunk,
|
||||
compressed_kv_cache=extra_kv_paged,
|
||||
out=output[query_start:query_end],
|
||||
bmm1_scale=self.scale,
|
||||
sinks=self.attn_sink,
|
||||
kv_layout="NHD",
|
||||
swa_topk_lens=swa_lens_chunk,
|
||||
extra_sparse_indices=extra_sparse_indices_chunk,
|
||||
extra_sparse_topk_lens=extra_sparse_lengths_chunk,
|
||||
)
|
||||
@@ -1,398 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWABackend,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import (
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
)
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
"""SWA metadata for the FlashMLA decode path, which allows varlen decode."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]:
|
||||
return DeepseekSparseSWAFlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
"""FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA)."""
|
||||
|
||||
backend_cls = DeepseekV4FlashMLABackend
|
||||
swa_backend_cls = DeepseekSparseSWAFlashMLABackend
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self.wo_b,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# FP8 decode kernel only supports h_q = 64 or 128.
|
||||
if num_heads > 128:
|
||||
raise ValueError(
|
||||
f"DeepseekV4 FlashMLA does not support {num_heads} heads "
|
||||
"(FP8 decode kernel requires h_q in {64, 128})."
|
||||
)
|
||||
return 64 if num_heads <= 64 else 128
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert output.shape == q.shape, (
|
||||
f"output buffer shape {output.shape} must match q shape {q.shape}"
|
||||
)
|
||||
assert output.dtype == q.dtype, (
|
||||
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
# Get SWA and indexer metadata from forward context
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: no real metadata. Reserve the same bf16
|
||||
# gather workspace _forward_prefill would; the dequantize / topk
|
||||
# / sparse_fwd kernels are skipped this step.
|
||||
swa_only = self.compress_ratio <= 1
|
||||
N = (
|
||||
0
|
||||
if swa_only
|
||||
else (self.max_model_len + self.compress_ratio - 1)
|
||||
// self.compress_ratio
|
||||
)
|
||||
M = N + self.window_size + self.max_num_batched_tokens
|
||||
if swa_only:
|
||||
top_k = 0
|
||||
else:
|
||||
assert self.topk_indices_buffer is not None
|
||||
top_k = self.topk_indices_buffer.shape[-1]
|
||||
combined_topk = round_up(
|
||||
top_k + self.window_size + self.max_image_tokens, 128
|
||||
)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
# SWA-only layers (compress_ratio <= 1) don't have their own KV cache
|
||||
# allocation, so self.kv_cache may be empty after profiling cleanup.
|
||||
self_kv_cache = self.kv_cache if not swa_only else None
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
|
||||
# Split prefill and decode
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
if num_prefills > 0:
|
||||
self._forward_prefill(
|
||||
q=q[num_decode_tokens:],
|
||||
positions=positions[num_decode_tokens:],
|
||||
compressed_k_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
output=output[num_decode_tokens:],
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
)
|
||||
if num_decodes > 0:
|
||||
self._forward_decode(
|
||||
q=q[:num_decode_tokens],
|
||||
kv_cache=self_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
def _forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None, # Only used when compress_ratio > 1
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
topk_indices = None
|
||||
topk_lens = None
|
||||
if not swa_only:
|
||||
assert attn_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if self.compress_ratio == 4:
|
||||
# C4A: local indices differ per layer (filled by Indexer).
|
||||
assert self.topk_indices_buffer is not None
|
||||
global_indices, topk_lens = compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
)
|
||||
topk_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
else:
|
||||
# C128A: pre-computed during metadata build.
|
||||
topk_indices = attn_metadata.c128a_global_decode_topk_indices
|
||||
topk_lens = attn_metadata.c128a_decode_topk_lens
|
||||
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
|
||||
# We treat queries in the same seq as different queries
|
||||
# and later we only attend by generated indices.
|
||||
# q arrives pre-padded to self.padded_heads by the outer wrapper.
|
||||
q = q.unsqueeze(1)
|
||||
|
||||
# Prepare SWA cache (num_blocks, swa_block_size, 1, head_bytes)
|
||||
# Use unsqueeze to preserve strides (handles padded blocks correctly)
|
||||
swa_cache = self.swa_cache_layer.kv_cache.unsqueeze(-2)
|
||||
# Reshape KV cache to (num_blocks, block_size, 1, head_bytes)
|
||||
if kv_cache is not None:
|
||||
kv_cache = kv_cache.unsqueeze(-2)
|
||||
|
||||
# One FlashMLASchedMeta per layer type, shared across all same-type
|
||||
# layers within this decode step. The first forward call per type
|
||||
# triggers the in-kernel planner (allocating tile_scheduler_metadata
|
||||
# and num_splits via PyTorch's graph-aware allocator so CUDA graph
|
||||
# capture reuses the same addresses on replay); subsequent same-type
|
||||
# layers see have_initialized=True and skip the planner.
|
||||
if self.compress_ratio <= 1:
|
||||
tile_metadata = swa_metadata.tile_sched_swaonly
|
||||
elif self.compress_ratio == 4:
|
||||
tile_metadata = swa_metadata.tile_sched_c4a
|
||||
elif self.compress_ratio == 128:
|
||||
tile_metadata = swa_metadata.tile_sched_c128a
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported compress_ratio={self.compress_ratio}; "
|
||||
"expected 1, 4, or 128."
|
||||
)
|
||||
assert tile_metadata is not None, (
|
||||
"swa_metadata missing tile_sched entry for "
|
||||
f"compress_ratio={self.compress_ratio}; "
|
||||
"DeepseekSparseSWAMetadataBuilder.build_tile_scheduler did not "
|
||||
"allocate one for this layer type."
|
||||
)
|
||||
|
||||
out, _ = flash_mla_with_kvcache(
|
||||
q=q,
|
||||
k_cache=swa_cache,
|
||||
block_table=None,
|
||||
head_dim_v=512,
|
||||
tile_scheduler_metadata=tile_metadata,
|
||||
cache_seqlens=None,
|
||||
is_fp8_kvcache=True,
|
||||
indices=swa_indices,
|
||||
topk_length=swa_lens,
|
||||
softmax_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
extra_k_cache=kv_cache if not swa_only else None,
|
||||
extra_indices_in_kvcache=topk_indices,
|
||||
extra_topk_length=topk_lens,
|
||||
out=output.unsqueeze(1),
|
||||
)
|
||||
|
||||
def _forward_prefill(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
compressed_k_cache: torch.Tensor | None, # Only used when compress_ratio > 1
|
||||
swa_k_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> None:
|
||||
swa_only = attn_metadata is None
|
||||
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
# Use pre-computed prefill metadata.
|
||||
seq_lens = swa_metadata.prefill_seq_lens
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
assert seq_lens is not None
|
||||
assert gather_lens is not None
|
||||
|
||||
# Derive prefill-local token offsets from the full query_start_loc_cpu.
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
assert query_start_loc_cpu is not None
|
||||
assert query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
|
||||
if not swa_only:
|
||||
if self.compress_ratio == 4:
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
|
||||
topk_indices = topk_indices[:num_prefill_tokens]
|
||||
else:
|
||||
# C128A: pre-computed during metadata build.
|
||||
assert attn_metadata is not None
|
||||
topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
top_k = topk_indices.shape[-1]
|
||||
else:
|
||||
# NOTE(woosuk): topk_indices will not be used for SWA-only layers.
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
|
||||
top_k = 0
|
||||
chunk_plan = swa_metadata.get_prefill_chunk_plan(
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
|
||||
)
|
||||
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
||||
workspace_manager = current_workspace_manager()
|
||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
||||
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
|
||||
chunk_size = chunk_end - chunk_start
|
||||
workspace = workspace_manager.get_simultaneous(
|
||||
((chunk_size, chunk_M, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
kv, combined_indices_out, combined_lens_out = workspace
|
||||
if not swa_only:
|
||||
# Gather compressed KV
|
||||
assert attn_metadata is not None
|
||||
block_table = attn_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
compressed_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
|
||||
gather_lens=None,
|
||||
block_table=block_table[chunk_start:chunk_end],
|
||||
block_size=attn_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
# Gather SWA KV
|
||||
swa_block_table = swa_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
swa_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end],
|
||||
gather_lens=gather_lens[chunk_start:chunk_end],
|
||||
block_table=swa_block_table[chunk_start:chunk_end],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=chunk_N,
|
||||
)
|
||||
|
||||
# Combine the topk indices and SWA indices for gathered KV cache
|
||||
query_start = (
|
||||
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
||||
)
|
||||
query_end = (
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
combined_indices_out = combined_indices_out[: query_end - query_start]
|
||||
combined_lens_out = combined_lens_out[: query_end - query_start]
|
||||
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices[query_start:query_end],
|
||||
query_start_loc[
|
||||
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
||||
],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
chunk_M,
|
||||
chunk_N,
|
||||
out=(combined_indices_out, combined_lens_out),
|
||||
left_visible=(
|
||||
swa_metadata.prefill_left_visible[
|
||||
num_decode_tokens + query_start : num_decode_tokens + query_end
|
||||
]
|
||||
if swa_metadata.prefill_left_visible is not None
|
||||
else None
|
||||
),
|
||||
right_visible=(
|
||||
swa_metadata.prefill_right_visible[
|
||||
num_decode_tokens + query_start : num_decode_tokens + query_end
|
||||
]
|
||||
if swa_metadata.prefill_right_visible is not None
|
||||
else None
|
||||
),
|
||||
max_image_tokens=self.max_image_tokens,
|
||||
)
|
||||
flash_mla_sparse_fwd(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
topk_length=combined_lens,
|
||||
out=output[query_start:query_end],
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,557 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
|
||||
|
||||
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
|
||||
pieces that have no analogue in V3/V32:
|
||||
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
|
||||
the fused ``eh_proj``);
|
||||
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
|
||||
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
|
||||
* V4-specific checkpoint weight-name remapping in ``load_weights``.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
# Other FP8 linear scales (including shared experts) always use
|
||||
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
|
||||
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
prefix: str,
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
|
||||
# fusing them the way V3 does with eh_proj.
|
||||
self.e_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.e_proj",
|
||||
)
|
||||
self.h_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.h_proj",
|
||||
)
|
||||
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.mtp_block = DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
|
||||
# reshape to (T, hc_mult, D) — the training-time layout — before
|
||||
# the fused norm pass so both inputs are 3D-friendly.
|
||||
previous_hidden_states = previous_hidden_states.view(
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight.data,
|
||||
self.hnorm.weight.data,
|
||||
self.enorm.variance_epsilon,
|
||||
self.hc_mult,
|
||||
)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, inputs_embeds
|
||||
)
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
previous_hidden_states = sp_shard(previous_hidden_states)
|
||||
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
mega_gate_metadata = None
|
||||
if self.use_native_mega_moe:
|
||||
routing_input_ids = input_ids
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
routing_input_ids = sp_shard(routing_input_ids)
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
routing_input_ids,
|
||||
has_hash_routing=False,
|
||||
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
|
||||
if getattr(self.config, "vision_n_layers", 0) > 0
|
||||
else None,
|
||||
)
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions,
|
||||
x=hidden_states,
|
||||
input_ids=input_ids,
|
||||
mega_gate_metadata=mega_gate_metadata,
|
||||
)
|
||||
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
|
||||
# Return the flat pre-hc_head residual so it can be re-fed as the
|
||||
# next spec step's `previous_hidden_states` when
|
||||
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
|
||||
return hidden_states.flatten(1)
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
|
||||
topk_tokens = config.index_topk
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
# Three aux streams shared across all MTP layers, mirroring DeepseekV4Model.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepSeekV4MultiTokenPredictorLayer(
|
||||
vllm_config,
|
||||
self.topk_indices_buffer,
|
||||
f"{prefix}.layers.{idx}",
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
|
||||
# hc_head here so logits are computed from the dense hidden state.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
|
||||
)
|
||||
hidden_states = hc_head_fused_kernel_tilelang(
|
||||
hidden_states,
|
||||
mtp_layer.hc_head_fn,
|
||||
mtp_layer.hc_head_scale,
|
||||
mtp_layer.hc_head_base,
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
)
|
||||
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
return logits
|
||||
|
||||
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.pad_shared_expert = getattr(
|
||||
self.quant_config, "weight_block_size", None
|
||||
) is not None and not _use_sequence_parallel(vllm_config)
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def _to_spec_layer_name(self, name: str) -> str:
|
||||
"""Remap V4 `mtp.{i}.*` names to `model.layers.{num_hidden_layers + i}.*`
|
||||
so that get_spec_layer_idx_from_weight_name can identify them."""
|
||||
mtp_layer_idx = 0
|
||||
for subname in name.split("."):
|
||||
try:
|
||||
# we use the first encountered integer
|
||||
mtp_layer_idx = int(subname)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
return name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
name = self._to_spec_layer_name(name)
|
||||
return get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
WEIGHT_NAME_REMAPPING: dict[str, str] = {
|
||||
".emb.tok_emb.weight": ".embed_tokens.weight",
|
||||
".head.weight": ".shared_head.head.weight",
|
||||
".norm.weight": ".shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def _remap_weight_name(name: str) -> str:
|
||||
"""Remap checkpoint weight names to model parameter names."""
|
||||
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
|
||||
if old_pattern in name:
|
||||
name = name.replace(old_pattern, new_pattern)
|
||||
return name
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_native_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
else:
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
|
||||
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
|
||||
# for the rename below based on the model's expert dtype.
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
name = self._to_spec_layer_name(name)
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
name = _remap_weight_name(name)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
|
||||
continue
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts." in name:
|
||||
experts = self.model.layers[str(spec_layer)].mtp_block.ffn.experts
|
||||
if getattr(experts, "has_fused_shared_experts", False):
|
||||
loaded_params.add(
|
||||
experts.load_shared_expert_weight(name, loaded_weight)
|
||||
)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
|
||||
self.quant_config, name, loaded_weight
|
||||
)
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# Reinterpret E8M0 scales as uint8 to preserve raw
|
||||
# exponent bytes; numeric copy_() would zero them.
|
||||
# Mirrors the main DeepseekV4 loader.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
else:
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
# ``e_score_correction_bias`` lives on the gate
|
||||
# under a different attribute name.
|
||||
name = name.replace(
|
||||
".ffn.gate.bias",
|
||||
".ffn.gate.e_score_correction_bias",
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may have "
|
||||
f"been quantized without including the MTP layers. "
|
||||
f"Use a checkpoint that includes MTP layer weights, "
|
||||
f"or disable speculative decoding."
|
||||
)
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"h_proj",
|
||||
"e_proj",
|
||||
"shared_head",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
# treat rest weights as weights for transformer layer block
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
# treat shared weights as top level weights
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NVIDIA-only (cutedsl/cutlass) kernels for DeepSeek V4.
|
||||
|
||||
These modules import ``cutlass``/``cutedsl`` at module top level, so they must
|
||||
not be imported on non-CUDA platforms. Callers should gate on
|
||||
``vllm.utils.import_utils.has_cutedsl()`` before importing from here.
|
||||
|
||||
This ``__init__`` deliberately imports nothing: re-exporting the cutedsl
|
||||
modules here would eagerly ``import cutlass`` (initializing the CUDA driver) for
|
||||
anyone who imports ``vllm.models.deepseek_v4``, breaking forked subprocesses.
|
||||
Import the leaf modules directly under a ``has_cutedsl()``/``is_cuda()`` gate.
|
||||
"""
|
||||
@@ -1,425 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Int32, Uint8, Uint32
|
||||
from cutlass.cute.nvgpu import cpasync
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import _bf16x2_mul, cvt
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
|
||||
|
||||
class DequantGatherKCacheKernel(
|
||||
VllmCuTeDSLJitKernel["DequantGatherKCacheKernel.CompileKey"]
|
||||
):
|
||||
# DSv4 geometry. The record format (how much of the row is quantized and
|
||||
# how wide a scale group is) comes from the compile key: V4 quantizes the
|
||||
# 448 NoPE dims in groups of 64 and keeps a bf16 RoPE tail, V4.1 quantizes
|
||||
# all 512 dims in groups of 32 (MXFP8) and has no tail.
|
||||
head_dim = 512
|
||||
num_warps = 4
|
||||
tb_size = num_warps * 32
|
||||
num_stages = 4
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
block_size: int
|
||||
has_gather_lens: bool
|
||||
fp8_dim: int = 448
|
||||
group_size: int = 64 # dims per scale
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
head_dim = DequantGatherKCacheKernel.head_dim
|
||||
fp8_dim = compile_key.fp8_dim
|
||||
group_size = compile_key.group_size
|
||||
num_warps = DequantGatherKCacheKernel.num_warps
|
||||
tb_size = DequantGatherKCacheKernel.tb_size
|
||||
num_stages = DequantGatherKCacheKernel.num_stages
|
||||
bf16_dim = head_dim - fp8_dim
|
||||
data_dim = fp8_dim + bf16_dim * 2
|
||||
scale_bytes = head_dim // group_size
|
||||
block_size = compile_key.block_size
|
||||
# V4.1 stores each token's scales right after its data; V4 puts all of
|
||||
# a block's data rows ahead of its scale rows.
|
||||
interleaved = bf16_dim == 0
|
||||
data_stride = data_dim + scale_bytes if interleaved else data_dim
|
||||
scale_offset = data_dim if interleaved else block_size * data_dim
|
||||
scale_stride = data_stride if interleaved else scale_bytes
|
||||
|
||||
@cute.jit
|
||||
def load_g2s(
|
||||
k_data_slice: cute.Tensor,
|
||||
k_scale: cute.Tensor,
|
||||
block_table: cute.Tensor,
|
||||
s_kdata_slice: cute.Tensor,
|
||||
s_kscale: cute.Tensor,
|
||||
req_id,
|
||||
pos,
|
||||
lane_id,
|
||||
stage_id,
|
||||
):
|
||||
op = cpasync.CopyG2SOp(cute.nvgpu.LoadCacheMode.GLOBAL)
|
||||
cp16_atom = cute.make_copy_atom(op, Uint32, num_bits_per_copy=128)
|
||||
cp_scale_atom = cute.make_copy_atom(
|
||||
cpasync.CopyG2SOp(), Uint8, num_bits_per_copy=scale_bytes * 8
|
||||
)
|
||||
page_id = block_table[req_id, pos // block_size]
|
||||
block_offset = pos % block_size
|
||||
|
||||
# Load the first 512 bytes (32x16B).
|
||||
idx = lane_id
|
||||
src = k_data_slice[page_id, block_offset, (None, idx)]
|
||||
cute.copy(
|
||||
cp16_atom,
|
||||
cute.recast_tensor(src, Uint32),
|
||||
s_kdata_slice[(None, idx), stage_id],
|
||||
)
|
||||
|
||||
# Load the tail 64 bytes.
|
||||
idx += 32
|
||||
if idx < cutlass.const_expr(data_dim // 16):
|
||||
src = k_data_slice[page_id, block_offset, (None, idx)]
|
||||
cute.copy(
|
||||
cp16_atom,
|
||||
cute.recast_tensor(src, Uint32),
|
||||
s_kdata_slice[(None, idx), stage_id],
|
||||
)
|
||||
elif idx == cutlass.const_expr(data_dim // 16):
|
||||
cute.copy(
|
||||
cp_scale_atom,
|
||||
k_scale[page_id, block_offset, None],
|
||||
s_kscale[None, stage_id],
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(
|
||||
out: cute.Tensor,
|
||||
k_data: cute.Tensor,
|
||||
k_scale: cute.Tensor,
|
||||
seq_lens: cute.Tensor,
|
||||
gather_lens: cute.Tensor | None,
|
||||
block_table: cute.Tensor,
|
||||
offset: Int32,
|
||||
):
|
||||
req_id, worker_id, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
_, num_workers, _ = cute.arch.grid_dim()
|
||||
|
||||
# Prepare smem.
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
s_kdata = smem.allocate_tensor(
|
||||
Uint32,
|
||||
cute.make_layout((data_dim // 4, num_warps, num_stages)),
|
||||
byte_alignment=16,
|
||||
)[None, warp_id, None]
|
||||
s_kscale = smem.allocate_tensor(
|
||||
Uint8,
|
||||
cute.make_layout((scale_bytes, num_warps, num_stages)),
|
||||
byte_alignment=scale_bytes,
|
||||
)[None, warp_id, None]
|
||||
|
||||
# Prepare for 16B cp.async, also for BF16 smem loads later.
|
||||
k_data_slice = cute.logical_divide(k_data, (None, None, 16))
|
||||
s_kdata_16B_slice = cute.logical_divide(s_kdata, (4, None))
|
||||
|
||||
# Load FP8 elems in 8B units, so once dequantized, they are 16B units.
|
||||
s_kdata_8B_slice = cute.logical_divide(s_kdata, (2, None))
|
||||
|
||||
# 16B st.global.
|
||||
out_slice = cute.logical_divide(out, (None, None, 8))
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp8_atom = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
|
||||
cp16_atom = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
|
||||
seq_len = seq_lens[req_id]
|
||||
gather_len = seq_len
|
||||
if cutlass.const_expr(gather_lens is not None):
|
||||
gather_len = gather_lens[req_id] # type: ignore[index]
|
||||
start_pos = seq_len - gather_len
|
||||
|
||||
# Start prefetch.
|
||||
for i in cutlass.range_constexpr(num_stages - 1):
|
||||
next_pos = (
|
||||
start_pos
|
||||
+ worker_id * num_warps
|
||||
+ warp_id
|
||||
+ i * num_workers * num_warps
|
||||
)
|
||||
if next_pos < seq_len:
|
||||
load_g2s(
|
||||
k_data_slice,
|
||||
k_scale,
|
||||
block_table,
|
||||
s_kdata_16B_slice,
|
||||
s_kscale,
|
||||
req_id,
|
||||
next_pos,
|
||||
lane_id,
|
||||
i,
|
||||
)
|
||||
cute.arch.cp_async_commit_group()
|
||||
prefetch_stage = num_stages - 1
|
||||
compute_stage = 0
|
||||
|
||||
# Main loop.
|
||||
for i in range(
|
||||
worker_id * num_warps + warp_id,
|
||||
gather_len,
|
||||
num_workers * num_warps,
|
||||
):
|
||||
pos = start_pos + i
|
||||
|
||||
# Prefetch next stage.
|
||||
next_pos = pos + num_workers * num_warps * (num_stages - 1)
|
||||
if next_pos < seq_len:
|
||||
load_g2s(
|
||||
k_data_slice,
|
||||
k_scale,
|
||||
block_table,
|
||||
s_kdata_16B_slice,
|
||||
s_kscale,
|
||||
req_id,
|
||||
next_pos,
|
||||
lane_id,
|
||||
prefetch_stage,
|
||||
)
|
||||
prefetch_stage = (prefetch_stage + 1) % num_stages
|
||||
cute.arch.cp_async_commit_group()
|
||||
|
||||
# Wait for gmem->smem to finish.
|
||||
cute.arch.cp_async_wait_group(num_stages - 1)
|
||||
cute.arch.sync_warp()
|
||||
|
||||
# There are 512 elems per token. As a warp, data0 holds the first
|
||||
# 256 elems and data1 holds the second 256 elems, i.e. each thread
|
||||
# holds 8 FP8 elems. This keeps the dequantized 8 BF16 elems as
|
||||
# contiguous 16B global stores. On Blackwell, this might not be
|
||||
# necessary as we have 32B global stores, but doing it this way
|
||||
# does not seem to be slower.
|
||||
data0 = cute.make_rmem_tensor((2,), Uint32)
|
||||
data1 = cute.make_rmem_tensor((2,), Uint32)
|
||||
cute.copy(
|
||||
cp8_atom, s_kdata_8B_slice[(None, lane_id), compute_stage], data0
|
||||
)
|
||||
cute.copy(
|
||||
cp8_atom,
|
||||
s_kdata_8B_slice[(None, lane_id + 32), compute_stage],
|
||||
data1,
|
||||
)
|
||||
|
||||
# Convert to bf16x2 via bit manipulation. FP8 scales are per 64
|
||||
# elements. An 8-element chunk advances the scale index by
|
||||
# chunk_id * 8 // group_size.
|
||||
scale0_u32 = Uint32(s_kscale[lane_id * 8 // group_size, compute_stage])
|
||||
scale0_bf16x2 = (scale0_u32 << Uint32(23)) | (scale0_u32 << Uint32(7))
|
||||
scale1_u32 = Uint32(
|
||||
s_kscale[(lane_id + 32) * 8 // group_size, compute_stage]
|
||||
)
|
||||
scale1_bf16x2 = (scale1_u32 << Uint32(23)) | (scale1_u32 << Uint32(7))
|
||||
|
||||
# cvt.rn.scaled::n2::ue8m0.bf16x2.e4m3x2 requires PTX 9.2
|
||||
# (CUDA 13.2).
|
||||
dequant0 = cute.make_rmem_tensor(4, Uint32)
|
||||
dequant1 = cute.make_rmem_tensor(4, Uint32)
|
||||
for j in cutlass.range_constexpr(2):
|
||||
tmp0 = cvt.fp8x4_to_bf16x4(data0[j])
|
||||
tmp1 = cvt.fp8x4_to_bf16x4(data1[j])
|
||||
|
||||
# BF16 multiply is safe because the scales are exact powers of 2.
|
||||
dequant0[j * 2] = _bf16x2_mul(tmp0[0], scale0_bf16x2)
|
||||
dequant1[j * 2] = _bf16x2_mul(tmp1[0], scale1_bf16x2)
|
||||
dequant0[j * 2 + 1] = _bf16x2_mul(tmp0[1], scale0_bf16x2)
|
||||
dequant1[j * 2 + 1] = _bf16x2_mul(tmp1[1], scale1_bf16x2)
|
||||
|
||||
# Last 64 elems are BF16 tail, corresponds to dequant1 of last
|
||||
# 8 threads. We have 448 FP8 + 64 BF16 -> 28x 16B for FP8 +
|
||||
# 8x 16B for BF16. The V4.1 record is fully quantized and has
|
||||
# no tail.
|
||||
if cutlass.const_expr(bf16_dim > 0): # noqa: SIM102
|
||||
if lane_id + 32 >= fp8_dim // 8:
|
||||
idx = fp8_dim // 16 + (lane_id + 32) - fp8_dim // 8
|
||||
cute.copy(
|
||||
cp16_atom,
|
||||
s_kdata_16B_slice[(None, idx), compute_stage],
|
||||
dequant1,
|
||||
)
|
||||
|
||||
# Store two 16B BF16 chunks per lane: first half, then second half.
|
||||
dst = out_slice[req_id, offset + i, (None, lane_id)]
|
||||
cute.copy(cp16_atom, dequant0, cute.recast_tensor(dst, Uint32))
|
||||
|
||||
dst = out_slice[req_id, offset + i, (None, lane_id + 32)]
|
||||
cute.copy(cp16_atom, dequant1, cute.recast_tensor(dst, Uint32))
|
||||
|
||||
compute_stage = (compute_stage + 1) % num_stages
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
out: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
seq_lens: cute.Tensor,
|
||||
gather_lens: cute.Tensor | None,
|
||||
block_table: cute.Tensor,
|
||||
offset: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
# Split k_cache into k_data [block_size, fp8_dim + bf16_dim * 2] and
|
||||
# k_scale [block_size, scale_bytes] views of each block.
|
||||
k_data = cute.make_tensor(
|
||||
k_cache.iterator,
|
||||
layout=cute.make_layout(
|
||||
(k_cache.shape[0], block_size, data_dim),
|
||||
stride=(k_cache.stride[0], data_stride, 1),
|
||||
),
|
||||
)
|
||||
# k_data_slice: [num_blocks, block_size, (16, data_dim/16)]
|
||||
# s_kdata_slice: [(4, data_dim/16), num_stages]
|
||||
k_scale = cute.make_tensor(
|
||||
k_cache.iterator + scale_offset,
|
||||
layout=cute.make_layout(
|
||||
(k_cache.shape[0], block_size, scale_bytes),
|
||||
stride=(k_cache.stride[0], scale_stride, 1),
|
||||
),
|
||||
)
|
||||
|
||||
num_reqs = out.shape[0]
|
||||
max_rows = out.shape[1] - offset
|
||||
num_workers = cutlass.max(
|
||||
1,
|
||||
cutlass.min(cute.ceil_div(max_rows, 4), cute.ceil_div(8192, num_reqs)),
|
||||
)
|
||||
grid = (num_reqs, num_workers, 1)
|
||||
device_kernel(
|
||||
out,
|
||||
k_data,
|
||||
k_scale,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
block_table,
|
||||
offset,
|
||||
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
block_size: int,
|
||||
has_gather_lens: bool,
|
||||
fp8_dim: int = 448,
|
||||
group_size: int = 64,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
block_size=block_size,
|
||||
has_gather_lens=has_gather_lens,
|
||||
fp8_dim=fp8_dim,
|
||||
group_size=group_size,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
dict(
|
||||
block_size=block_size,
|
||||
has_gather_lens=True,
|
||||
enabled=True,
|
||||
),
|
||||
dict(
|
||||
block_size=max(1, block_size // 4),
|
||||
has_gather_lens=False,
|
||||
enabled=4 in compress_ratios,
|
||||
),
|
||||
dict(
|
||||
block_size=max(1, block_size // 128),
|
||||
has_gather_lens=False,
|
||||
enabled=128 in compress_ratios,
|
||||
),
|
||||
),
|
||||
valid_block_size=block_size > 0,
|
||||
_when=lambda *, enabled, valid_block_size: enabled and valid_block_size,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_reqs = cute.sym_int()
|
||||
head_dim = self.head_dim
|
||||
fp8_dim = compile_key.fp8_dim
|
||||
block_size = compile_key.block_size
|
||||
head_bytes = (
|
||||
fp8_dim + (head_dim - fp8_dim) * 2 + head_dim // compile_key.group_size
|
||||
)
|
||||
|
||||
out = make_fake_tensor(BFloat16, (num_reqs, cute.sym_int(), head_dim), 16)
|
||||
k_cache = cute.runtime.make_fake_tensor(
|
||||
Uint8,
|
||||
(cute.sym_int(), block_size, head_bytes),
|
||||
stride=(cute.sym_int64(divisibility=32), head_bytes, 1),
|
||||
assumed_align=32,
|
||||
)
|
||||
seq_lens = make_fake_tensor(Int32, (num_reqs,))
|
||||
gather_lens = (
|
||||
make_fake_tensor(Int32, (num_reqs,))
|
||||
if compile_key.has_gather_lens
|
||||
else None
|
||||
)
|
||||
block_table = make_fake_tensor(Int32, (num_reqs, cute.sym_int()))
|
||||
|
||||
return (
|
||||
out,
|
||||
k_cache,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
block_table,
|
||||
Int32(0),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
gather_lens: torch.Tensor | None,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
offset: int,
|
||||
fp8_dim: int = 448,
|
||||
group_size: int = 64,
|
||||
) -> CuTeDSLLaunchSpec["DequantGatherKCacheKernel.CompileKey"]:
|
||||
compile_key = self.dispatch(
|
||||
block_size=block_size,
|
||||
has_gather_lens=gather_lens is not None,
|
||||
fp8_dim=fp8_dim,
|
||||
group_size=group_size,
|
||||
)
|
||||
launch_args = (
|
||||
out,
|
||||
k_cache,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
block_table,
|
||||
offset,
|
||||
)
|
||||
return compile_key, launch_args
|
||||
|
||||
|
||||
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL = DequantGatherKCacheKernel()
|
||||
@@ -1,708 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int64, Uint8, Uint32, const_expr
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
_bf16x2_abs,
|
||||
_bf16x2_max,
|
||||
cvt,
|
||||
recast_val,
|
||||
torch_to_cute_dtype,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.vllm_flash_attn.cute import utils as cute_utils
|
||||
|
||||
# MXFP4: 32 elements per block, packed 2 nibbles per byte, ue8m0 block scale.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _load_q_and_rope(
|
||||
positions: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
num_heads: cutlass.Constexpr,
|
||||
coarsen: cutlass.Constexpr,
|
||||
tb_size: cutlass.Constexpr,
|
||||
subwarp_size: cutlass.Constexpr,
|
||||
threads_per_token: cutlass.Constexpr,
|
||||
nope_dim: cutlass.Constexpr,
|
||||
rope_dim: cutlass.Constexpr,
|
||||
cos_sin_dtype: cutlass.Constexpr,
|
||||
):
|
||||
"""Compute thread indices, load Q (BF16), and apply interleaved RoPE.
|
||||
|
||||
Returns a tuple
|
||||
(q_bf16x2, tid, global_tid, sublane, token_id, head_tile_id,
|
||||
head_start, in_bounds, num_token_heads)
|
||||
where ``q_bf16x2`` is a (coarsen, 8) rmem tile of Uint32 packed
|
||||
bf16x2 pairs covering the 16 BF16 lanes owned by this thread for
|
||||
each of ``coarsen`` heads. RoPE is applied in place to the
|
||||
trailing ``rope_dim`` lanes; the leading nope lanes pass through.
|
||||
"""
|
||||
block_id, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
|
||||
num_tokens = q.shape[0]
|
||||
num_token_heads = num_tokens * num_heads
|
||||
global_tid = block_id * tb_size + tid
|
||||
|
||||
global_subwarp_id = global_tid // subwarp_size
|
||||
sublane = tid % subwarp_size
|
||||
|
||||
token_id = global_subwarp_id // (num_heads // coarsen)
|
||||
head_tile_id = global_subwarp_id % (num_heads // coarsen)
|
||||
head_start = head_tile_id * coarsen
|
||||
|
||||
# NOTE: token_id may exceed bounds, hence we need to add load/store guards
|
||||
# we can't do early exit because CuteDSL doesn't support it. and we also need
|
||||
# all threads in a warp to be active since we utilize warp shuffle later.
|
||||
# must_in_bounds is constexpr, True when a token spans a whole number of
|
||||
# threadblocks. Then num_tokens * threads_per_token is always a multiple of
|
||||
# tb_size, so the trailing block never straddles num_tokens and the compiler
|
||||
# removes the bounds check. Otherwise the guard has to stay: e.g. 32 heads
|
||||
# with coarsen=4 needs 64 threads per token, so an odd num_tokens leaves half
|
||||
# of the last block addressing token num_tokens.
|
||||
must_in_bounds = cutlass.const_expr(threads_per_token % tb_size == 0)
|
||||
in_bounds = must_in_bounds or (token_id < num_tokens)
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
|
||||
_layout = cute.make_layout((coarsen, 8), stride=(8, 1))
|
||||
q_bf16x2 = cute.make_rmem_tensor(_layout, Uint32)
|
||||
|
||||
if in_bounds:
|
||||
# we can't do cute.copy() on the whole 2D tile directly because
|
||||
# cute.copy() wants the 1st mode to be covered by the copy atom,
|
||||
# and other modes as for loop. there is no fast way to
|
||||
# "transpose" the tensor view.
|
||||
q_tile = cute.local_tile(
|
||||
q[token_id, None, None],
|
||||
tiler=(coarsen, 16),
|
||||
coord=(head_tile_id, sublane),
|
||||
)
|
||||
cp_u32x8 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
|
||||
for i in cutlass.range_constexpr(coarsen):
|
||||
src = cute.recast_tensor(q_tile[i, None], Uint32)
|
||||
cute.copy(cp_u32x8, src, q_bf16x2[i, None])
|
||||
|
||||
# RoPE applies only to the trailing rope_dim values. We keep the rounded
|
||||
# BF16 result in q_bits so the later amax and quantization see BF16.
|
||||
# cos_sin_cache layout: [max_pos, rope_dim]
|
||||
if in_bounds and sublane * 16 >= nope_dim:
|
||||
cos_vals = cute.make_rmem_tensor((8,), Float32)
|
||||
sin_vals = cute.make_rmem_tensor((8,), Float32)
|
||||
|
||||
pos = positions[token_id]
|
||||
|
||||
# select 8 elems from cos and sin
|
||||
cos_id = sublane - nope_dim // 16
|
||||
sin_id = cos_id + rope_dim // 16
|
||||
cos_src = cute.local_tile(cos_sin_cache[pos, None], tiler=(8,), coord=(cos_id,))
|
||||
sin_src = cute.local_tile(cos_sin_cache[pos, None], tiler=(8,), coord=(sin_id,))
|
||||
|
||||
cp_f32x8 = cute.make_copy_atom(cp_op, Float32, num_bits_per_copy=256)
|
||||
cp_u32x4 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
|
||||
if const_expr(cos_sin_dtype is Float32):
|
||||
cute.copy(cp_f32x8, cos_src, cos_vals)
|
||||
cute.copy(cp_f32x8, sin_src, sin_vals)
|
||||
else:
|
||||
cos_bf16x2 = cute.make_rmem_tensor((4,), Uint32)
|
||||
sin_bf16x2 = cute.make_rmem_tensor((4,), Uint32)
|
||||
cute.copy(cp_u32x4, cute.recast_tensor(cos_src, Uint32), cos_bf16x2)
|
||||
cute.copy(cp_u32x4, cute.recast_tensor(sin_src, Uint32), sin_bf16x2)
|
||||
|
||||
for i in cutlass.range_constexpr(4):
|
||||
cos0, cos1 = cvt.bf16x2_to_fp32x2(cos_bf16x2[i])
|
||||
sin0, sin1 = cvt.bf16x2_to_fp32x2(sin_bf16x2[i])
|
||||
cos_vals[i * 2] = cos0
|
||||
cos_vals[i * 2 + 1] = cos1
|
||||
sin_vals[i * 2] = sin0
|
||||
sin_vals[i * 2 + 1] = sin1
|
||||
|
||||
for i in cutlass.range_constexpr(coarsen):
|
||||
for j in cutlass.range_constexpr(8):
|
||||
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j])
|
||||
rot0 = q0 * cos_vals[j] - q1 * sin_vals[j]
|
||||
rot1 = q0 * sin_vals[j] + q1 * cos_vals[j]
|
||||
# convert back to BF16 to match numerics
|
||||
q_bf16x2[i, j] = cvt.fp32x2_to_bf16x2(rot0, rot1)
|
||||
|
||||
return (
|
||||
q_bf16x2,
|
||||
tid,
|
||||
global_tid,
|
||||
sublane,
|
||||
token_id,
|
||||
head_tile_id,
|
||||
head_start,
|
||||
in_bounds,
|
||||
num_token_heads,
|
||||
)
|
||||
|
||||
|
||||
class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]):
|
||||
tb_size = 128
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
head_dim: int
|
||||
rope_dim: int
|
||||
num_heads: int
|
||||
cos_sin_dtype: type[cutlass.Numeric]
|
||||
weights_out_dtype: type[cutlass.Numeric]
|
||||
coarsen: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
head_dim = compile_key.head_dim
|
||||
rope_dim = compile_key.rope_dim
|
||||
nope_dim = head_dim - rope_dim
|
||||
num_heads = compile_key.num_heads
|
||||
cos_sin_dtype = compile_key.cos_sin_dtype
|
||||
# process multiple heads at the same time to armotize RoPE load costs
|
||||
coarsen = compile_key.coarsen
|
||||
tb_size = IndexerQMxFp4Kernel.tb_size
|
||||
weights_out_dtype = compile_key.weights_out_dtype
|
||||
assert num_heads % coarsen == 0
|
||||
|
||||
# later we will use 32B load = 16 BF16 elems
|
||||
# thus, head_dim=128 requires 8 threads to handle.
|
||||
# let's call subwarp = 8 threads.
|
||||
subwarp_size = head_dim // 16
|
||||
threads_per_token = (num_heads // coarsen) * subwarp_size
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(
|
||||
positions: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
weights: cute.Tensor,
|
||||
q_quant: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
weights_out: cute.Tensor,
|
||||
scale: Float32,
|
||||
):
|
||||
(
|
||||
q_bf16x2,
|
||||
tid,
|
||||
global_tid,
|
||||
sublane,
|
||||
token_id,
|
||||
head_tile_id,
|
||||
head_start,
|
||||
in_bounds,
|
||||
num_token_heads,
|
||||
) = _load_q_and_rope(
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
num_heads,
|
||||
coarsen,
|
||||
tb_size,
|
||||
subwarp_size,
|
||||
threads_per_token,
|
||||
nope_dim,
|
||||
rope_dim,
|
||||
cos_sin_dtype,
|
||||
)
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
|
||||
# layout: [coarsen, 8]
|
||||
q_fp4_tile = cute.local_tile(
|
||||
q_quant[token_id, None, None],
|
||||
tiler=(coarsen, 8),
|
||||
coord=(head_tile_id, sublane),
|
||||
)
|
||||
|
||||
for i in cutlass.range_constexpr(coarsen):
|
||||
# compute amax in packed bf16x2 to save instructions
|
||||
# Each thread holds 16 elems. Two adjacent threads form one 32-elem
|
||||
# MXFP4 block, so a width-2 shuffle gives the block amax.
|
||||
amax_bf16x2 = _bf16x2_abs(q_bf16x2[i, 0])
|
||||
for j in cutlass.range_constexpr(1, 8):
|
||||
amax_bf16x2 = _bf16x2_max(amax_bf16x2, _bf16x2_abs(q_bf16x2[i, j]))
|
||||
amax_bf16x2 = cute_utils.warp_reduce(
|
||||
amax_bf16x2,
|
||||
_bf16x2_max,
|
||||
width=MXFP4_BLOCK_SIZE // 16,
|
||||
)
|
||||
amax_pair = cvt.bf16x2_to_fp32x2(amax_bf16x2)
|
||||
amax = cute_utils.fmax(amax_pair[0], amax_pair[1])
|
||||
|
||||
if in_bounds:
|
||||
# compute block scale with bit manipulation
|
||||
# UE8M0 stores ceil(log2(fp4_scale)) + 127. Adding the mantissa mask
|
||||
# increments the exponent whenever fp4_scale is not exactly a power of 2
|
||||
eps = cutlass.const_expr(float.fromhex("0x6p-126"))
|
||||
fp4_scale = cute_utils.fmax(amax, eps) * Float32(1.0 / 6.0)
|
||||
bits = recast_val(fp4_scale, Uint32)
|
||||
ue8m0 = cute_utils.shr_u32(
|
||||
bits + Uint32(0x7FFFFF), Uint32(23)
|
||||
) & Uint32(0xFF)
|
||||
|
||||
# Only one of the two threads in an MXFP4 block writes the shared scale.
|
||||
if tid % 2 == 0:
|
||||
mx_block = sublane // 2
|
||||
q_scale[token_id, head_start + i, mx_block] = Uint8(ue8m0)
|
||||
|
||||
# If scale = 2^A and ue8m0 = A + 127, then inverse scale has exponent
|
||||
# -A + 127 = 254 - ue8m0.
|
||||
inv_scale_bits = (Uint32(254) - ue8m0) << Uint32(23)
|
||||
inv_fp4_scale = recast_val(inv_scale_bits, Float32)
|
||||
|
||||
vals = cute.make_rmem_tensor(16, Float32)
|
||||
for j in cutlass.range_constexpr(8):
|
||||
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j])
|
||||
vals[j * 2] = q0 * inv_fp4_scale
|
||||
vals[j * 2 + 1] = q1 * inv_fp4_scale
|
||||
|
||||
# pack to FP4
|
||||
packed = cute.make_rmem_tensor((2,), Uint32)
|
||||
packed[0] = cvt.fp32x8_to_fp4x8(vals, 0)
|
||||
packed[1] = cvt.fp32x8_to_fp4x8(vals, 8)
|
||||
|
||||
dst = q_fp4_tile[i, None]
|
||||
cp_u32x2 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
|
||||
cute.copy(cp_u32x2, packed, cute.recast_tensor(dst, Uint32))
|
||||
|
||||
# Weight scaling is independent of the Q subwarp work. The first
|
||||
# num_tokens * num_heads logical threads cover one weight each.
|
||||
if global_tid < num_token_heads:
|
||||
weight_token_id = global_tid // num_heads
|
||||
weight_head_id = global_tid % num_heads
|
||||
weights_out[weight_token_id, weight_head_id] = (
|
||||
weights[weight_token_id, weight_head_id].to(Float32) * scale
|
||||
).to(weights_out_dtype)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
positions: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
weights: cute.Tensor,
|
||||
q_quant: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
weights_out: cute.Tensor,
|
||||
scale: Float32,
|
||||
stream: CUstream,
|
||||
):
|
||||
total_threads = q.shape[0] * threads_per_token
|
||||
grid = (cute.ceil_div(total_threads, tb_size), 1, 1)
|
||||
device_kernel(
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_quant,
|
||||
q_scale,
|
||||
weights_out,
|
||||
scale,
|
||||
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
cos_sin_dtype: type[cutlass.Numeric],
|
||||
weights_out_dtype: type[cutlass.Numeric],
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
cos_sin_dtype=cos_sin_dtype,
|
||||
weights_out_dtype=weights_out_dtype,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
from vllm.v1.attention.backends.mla.indexer import dsa_indexer_uses_fp4
|
||||
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if hf_config is None or not dsa_indexer_uses_fp4(vllm_config):
|
||||
return []
|
||||
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
# The DeepSeek V4.1 sparse-logits indexer takes bf16 weights; the
|
||||
# dense scoring kernels take fp32.
|
||||
weights_out_dtypes: tuple[type[cutlass.Numeric], ...] = (Float32,)
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
weights_out_dtypes = (Float32, BFloat16)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=Float32,
|
||||
weights_out_dtype=weights_out_dtypes,
|
||||
coarsen=(1, 4),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_tokens = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
q = make_fake_tensor(
|
||||
BFloat16,
|
||||
(num_tokens, compile_key.num_heads, compile_key.head_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
positions = make_fake_tensor(Int64, (num_tokens,), divisibility=1)
|
||||
cos_sin_cache = make_fake_tensor(
|
||||
compile_key.cos_sin_dtype,
|
||||
(max_pos, compile_key.rope_dim),
|
||||
divisibility=8,
|
||||
)
|
||||
weights = make_fake_tensor(
|
||||
BFloat16, (num_tokens, compile_key.num_heads), divisibility=8
|
||||
)
|
||||
q_packed = make_fake_tensor(
|
||||
Uint8,
|
||||
(num_tokens, compile_key.num_heads, compile_key.head_dim // 2),
|
||||
divisibility=16,
|
||||
)
|
||||
q_scale = make_fake_tensor(
|
||||
Uint8,
|
||||
(
|
||||
num_tokens,
|
||||
compile_key.num_heads,
|
||||
compile_key.head_dim // MXFP4_BLOCK_SIZE,
|
||||
),
|
||||
divisibility=4,
|
||||
)
|
||||
weights_out = make_fake_tensor(
|
||||
compile_key.weights_out_dtype,
|
||||
(num_tokens, compile_key.num_heads),
|
||||
divisibility=4,
|
||||
)
|
||||
return (
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_packed,
|
||||
q_scale,
|
||||
weights_out,
|
||||
Float32(0.0),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
weights_softmax_scale: float,
|
||||
weights_head_scale: float,
|
||||
q_packed: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
weights_out: torch.Tensor,
|
||||
) -> CuTeDSLLaunchSpec["IndexerQMxFp4Kernel.CompileKey"]:
|
||||
num_tokens, num_heads, head_dim = q.shape
|
||||
compile_key = self.dispatch(
|
||||
head_dim=head_dim,
|
||||
rope_dim=cos_sin_cache.shape[-1],
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
|
||||
weights_out_dtype=torch_to_cute_dtype(weights_out.dtype),
|
||||
coarsen=1 if num_tokens < 512 else 4,
|
||||
)
|
||||
launch_args = (
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_packed,
|
||||
q_scale,
|
||||
weights_out,
|
||||
float(weights_softmax_scale * weights_head_scale),
|
||||
)
|
||||
return compile_key, launch_args
|
||||
|
||||
|
||||
class IndexerQFp8Kernel(VllmCuTeDSLJitKernel["IndexerQFp8Kernel.CompileKey"]):
|
||||
tb_size = 128
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
head_dim: int
|
||||
rope_dim: int
|
||||
num_heads: int
|
||||
cos_sin_dtype: type[cutlass.Numeric]
|
||||
coarsen: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
head_dim = compile_key.head_dim
|
||||
rope_dim = compile_key.rope_dim
|
||||
nope_dim = head_dim - rope_dim
|
||||
num_heads = compile_key.num_heads
|
||||
cos_sin_dtype = compile_key.cos_sin_dtype
|
||||
# process multiple heads at the same time to armotize RoPE load costs
|
||||
coarsen = compile_key.coarsen
|
||||
tb_size = IndexerQFp8Kernel.tb_size
|
||||
assert num_heads % coarsen == 0
|
||||
|
||||
# later we will use 32B load = 16 BF16 elems
|
||||
# thus, head_dim=128 requires 8 threads to handle.
|
||||
# let's call subwarp = 8 threads.
|
||||
subwarp_size = head_dim // 16
|
||||
threads_per_token = (num_heads // coarsen) * subwarp_size
|
||||
# Each subwarp owns `coarsen` heads; we use the first `coarsen`
|
||||
# threads of the subwarp to write the per-head weights using the
|
||||
# fp8 scale computed in the matching loop iteration.
|
||||
assert coarsen <= subwarp_size, (
|
||||
f"FP8 kernel requires coarsen ({coarsen}) <= "
|
||||
f"subwarp_size ({subwarp_size}) for the weight-fold step"
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(
|
||||
positions: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
weights: cute.Tensor,
|
||||
q_fp8: cute.Tensor,
|
||||
weights_out: cute.Tensor,
|
||||
scale: Float32,
|
||||
):
|
||||
(
|
||||
q_bf16x2,
|
||||
_tid,
|
||||
_global_tid,
|
||||
sublane,
|
||||
token_id,
|
||||
head_tile_id,
|
||||
head_start,
|
||||
in_bounds,
|
||||
_num_token_heads,
|
||||
) = _load_q_and_rope(
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
num_heads,
|
||||
coarsen,
|
||||
tb_size,
|
||||
subwarp_size,
|
||||
threads_per_token,
|
||||
nope_dim,
|
||||
rope_dim,
|
||||
cos_sin_dtype,
|
||||
)
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
|
||||
# layout: [coarsen, 16] bytes (one e4m3fn per element).
|
||||
q_fp8_tile = cute.local_tile(
|
||||
q_fp8[token_id, None, None],
|
||||
tiler=(coarsen, 16),
|
||||
coord=(head_tile_id, sublane),
|
||||
)
|
||||
|
||||
for i in cutlass.range_constexpr(coarsen):
|
||||
# Reduce amax across the full head_dim: each thread already holds
|
||||
# the max over its 16 lanes; a width=subwarp_size warp shuffle
|
||||
# spreads the head-wide max to every lane in the subwarp.
|
||||
amax_bf16x2 = _bf16x2_abs(q_bf16x2[i, 0])
|
||||
for j in cutlass.range_constexpr(1, 8):
|
||||
amax_bf16x2 = _bf16x2_max(amax_bf16x2, _bf16x2_abs(q_bf16x2[i, j]))
|
||||
amax_bf16x2 = cute_utils.warp_reduce(
|
||||
amax_bf16x2,
|
||||
_bf16x2_max,
|
||||
width=subwarp_size,
|
||||
)
|
||||
amax_pair = cvt.bf16x2_to_fp32x2(amax_bf16x2)
|
||||
amax = cute_utils.fmax(amax_pair[0], amax_pair[1])
|
||||
|
||||
# scale = max(amax, eps) / fp8_max, then rounded UP to the next
|
||||
# power of two. Adding the mantissa mask before shifting out the
|
||||
# mantissa bumps the exponent whenever s isn't a pure pow2.
|
||||
fp32_scale = cute_utils.fmax(amax, Float32(1e-4)) * Float32(1.0 / 448.0)
|
||||
bits = recast_val(fp32_scale, Uint32)
|
||||
scale_exp = cute_utils.shr_u32(
|
||||
bits + Uint32(0x7FFFFF), Uint32(23)
|
||||
) & Uint32(0xFF)
|
||||
|
||||
# rounded scale = 2^(scale_exp - 127); bit pattern is scale_exp << 23
|
||||
fp8_scale_bits = scale_exp << Uint32(23)
|
||||
fp8_scale = recast_val(fp8_scale_bits, Float32)
|
||||
# inverse = 2^-(scale_exp - 127); bit pattern is (254 - scale_exp) << 23
|
||||
inv_scale_bits = (Uint32(254) - scale_exp) << Uint32(23)
|
||||
inv_fp8_scale = recast_val(inv_scale_bits, Float32)
|
||||
|
||||
# Weight fold: weights_out = weights * q_scale * scale_combined.
|
||||
# All threads in the subwarp share the same fp8_scale after the
|
||||
# warp_reduce above, so we let thread `sublane == i` write the
|
||||
# weight for head `head_start + i`.
|
||||
if in_bounds and sublane == i:
|
||||
head_id = head_start + i
|
||||
weights_out[token_id, head_id] = (
|
||||
weights[token_id, head_id].to(Float32) * scale * fp8_scale
|
||||
)
|
||||
|
||||
if in_bounds:
|
||||
# 16 BF16 → 16 e4m3 bytes per thread, packed into 4 b32s
|
||||
# (one cp.async-shaped 128-bit store per row).
|
||||
packed = cute.make_rmem_tensor((4,), Uint32)
|
||||
for j in cutlass.range_constexpr(4):
|
||||
q0, q1 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j * 2])
|
||||
q2, q3 = cvt.bf16x2_to_fp32x2(q_bf16x2[i, j * 2 + 1])
|
||||
packed[j] = cvt.fp32x4_to_fp8x4(
|
||||
q0 * inv_fp8_scale,
|
||||
q1 * inv_fp8_scale,
|
||||
q2 * inv_fp8_scale,
|
||||
q3 * inv_fp8_scale,
|
||||
)
|
||||
|
||||
dst = q_fp8_tile[i, None]
|
||||
cp_u32x4 = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cute.copy(cp_u32x4, packed, cute.recast_tensor(dst, Uint32))
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
positions: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
weights: cute.Tensor,
|
||||
q_fp8: cute.Tensor,
|
||||
weights_out: cute.Tensor,
|
||||
scale: Float32,
|
||||
stream: CUstream,
|
||||
):
|
||||
total_threads = q.shape[0] * threads_per_token
|
||||
grid = (cute.ceil_div(total_threads, tb_size), 1, 1)
|
||||
device_kernel(
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_fp8,
|
||||
weights_out,
|
||||
scale,
|
||||
).launch(grid=grid, block=(tb_size, 1, 1), stream=stream)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
cos_sin_dtype: type[cutlass.Numeric],
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
cos_sin_dtype=cos_sin_dtype,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
from vllm.v1.attention.backends.mla.indexer import dsa_indexer_uses_fp4
|
||||
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if hf_config is None or dsa_indexer_uses_fp4(vllm_config):
|
||||
return []
|
||||
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=Float32,
|
||||
coarsen=(1, 4),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_tokens = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
q = make_fake_tensor(
|
||||
BFloat16,
|
||||
(num_tokens, compile_key.num_heads, compile_key.head_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
positions = make_fake_tensor(Int64, (num_tokens,), divisibility=1)
|
||||
cos_sin_cache = make_fake_tensor(
|
||||
compile_key.cos_sin_dtype,
|
||||
(max_pos, compile_key.rope_dim),
|
||||
divisibility=8,
|
||||
)
|
||||
weights = make_fake_tensor(
|
||||
BFloat16, (num_tokens, compile_key.num_heads), divisibility=8
|
||||
)
|
||||
q_fp8 = make_fake_tensor(
|
||||
Uint8,
|
||||
(num_tokens, compile_key.num_heads, compile_key.head_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
weights_out = make_fake_tensor(
|
||||
Float32, (num_tokens, compile_key.num_heads), divisibility=4
|
||||
)
|
||||
return (
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_fp8,
|
||||
weights_out,
|
||||
Float32(0.0),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
weights_softmax_scale: float,
|
||||
weights_head_scale: float,
|
||||
q_fp8: torch.Tensor,
|
||||
weights_out: torch.Tensor,
|
||||
) -> CuTeDSLLaunchSpec["IndexerQFp8Kernel.CompileKey"]:
|
||||
num_tokens, num_heads, head_dim = q.shape
|
||||
compile_key = self.dispatch(
|
||||
head_dim=head_dim,
|
||||
rope_dim=cos_sin_cache.shape[-1],
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
|
||||
coarsen=1 if num_tokens < 512 else 4,
|
||||
)
|
||||
launch_args = (
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
q_fp8.view(torch.uint8),
|
||||
weights_out,
|
||||
float(weights_softmax_scale * weights_head_scale),
|
||||
)
|
||||
return compile_key, launch_args
|
||||
|
||||
|
||||
_INDEXER_Q_MXFP4_KERNEL = IndexerQMxFp4Kernel()
|
||||
_INDEXER_Q_FP8_KERNEL = IndexerQFp8Kernel()
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
|
||||
|
||||
|
||||
def compute_fp8_einsum_recipe(
|
||||
block_size: int = 128,
|
||||
) -> tuple[tuple[int, int, int], bool]:
|
||||
"""fp8_einsum recipe + scale layout for the current GPU arch.
|
||||
|
||||
SM90 keeps block-row FP32 scales. SM100 uses packed per-row E8M0 scales.
|
||||
|
||||
Returns ``(einsum_recipe, tma_aligned_scales)`` for ``deep_gemm_fp8_o_proj``.
|
||||
"""
|
||||
cap = current_platform.get_device_capability()
|
||||
assert cap is not None, "DeepseekV4 attention requires a CUDA device"
|
||||
einsum_recipe = (1, 128, 128) if cap.major <= 9 else (1, 1, block_size)
|
||||
tma_aligned_scales = cap.major >= 10
|
||||
return einsum_recipe, tma_aligned_scales
|
||||
|
||||
|
||||
def deep_gemm_fp8_o_proj(
|
||||
o: torch.Tensor | QuantizedActivation,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
wo_a: nn.Module,
|
||||
wo_b: Callable[[torch.Tensor], torch.Tensor],
|
||||
*,
|
||||
n_groups: int,
|
||||
heads_per_group: int,
|
||||
nope_dim: int,
|
||||
rope_dim: int,
|
||||
o_lora_rank: int,
|
||||
einsum_recipe: tuple[int, int, int],
|
||||
tma_aligned_scales: bool,
|
||||
) -> torch.Tensor:
|
||||
"""O projection: inverse RoPE + grouped wo_a + wo_b.
|
||||
|
||||
Shared by the FlashMLA and FlashInfer CUDA backends. The attention
|
||||
layer selects the recipe at initialization. ``wo_b`` is any callable over
|
||||
the flattened ``z``: the projection module itself, or a wrapper that also
|
||||
reduce-scatters its output (DeepSeek-V4.1 GEMM-RS).
|
||||
|
||||
A QuantizedActivation ``o`` was already rotated and cast by the attention
|
||||
kernel, one ``wo_a`` group per slot with the live groups first, so only
|
||||
the padding groups are sliced off.
|
||||
"""
|
||||
use_fp8 = wo_a.weight.dtype == torch.float8_e4m3fn
|
||||
if isinstance(o, QuantizedActivation):
|
||||
assert use_fp8
|
||||
o_proj_input, o_scale = o.data[:, :n_groups], o.scale[:, :n_groups]
|
||||
else:
|
||||
o_proj_input, o_scale = fused_inv_rope_fp8_quant(
|
||||
o,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups=n_groups,
|
||||
heads_per_group=heads_per_group,
|
||||
nope_dim=nope_dim,
|
||||
rope_dim=rope_dim,
|
||||
quant_group_size=einsum_recipe[2],
|
||||
tma_aligned_scales=tma_aligned_scales,
|
||||
quantize=use_fp8,
|
||||
)
|
||||
z = torch.empty(
|
||||
(o_proj_input.shape[0], n_groups, o_lora_rank),
|
||||
device=o_proj_input.device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if use_fp8:
|
||||
weight_scale = (
|
||||
wo_a.weight_scale
|
||||
if hasattr(wo_a, "weight_scale")
|
||||
else wo_a.weight_scale_inv
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(o_proj_input, o_scale),
|
||||
(wo_a.weight, weight_scale),
|
||||
z,
|
||||
recipe=einsum_recipe,
|
||||
)
|
||||
else:
|
||||
grouped_weight = wo_a.weight.view(n_groups, o_lora_rank, -1)
|
||||
torch.bmm(
|
||||
o_proj_input.transpose(0, 1),
|
||||
grouped_weight.transpose(1, 2),
|
||||
out=z.transpose(0, 1),
|
||||
)
|
||||
return wo_b(z.flatten(1))
|
||||
|
||||
|
||||
# TRTLLM-gen's DSv4 RopeQuant epilogue (flashinfer-ai/flashinfer#4918) has one
|
||||
# fixed schedule: 128 query heads in groups of 8 512-wide heads, and FP8 with
|
||||
# one packed UE8M0 scale per 128 elements.
|
||||
_ROPE_QUANT_NUM_HEADS = 128
|
||||
_ROPE_QUANT_HEADS_PER_GROUP = 8
|
||||
_ROPE_QUANT_HEAD_DIM = 512
|
||||
_ROPE_QUANT_BLOCK = 128
|
||||
|
||||
|
||||
def rope_quant_unsupported_reason(layer: torch.nn.Module) -> str | None:
|
||||
"""Why ``layer`` cannot take FlashInfer's fused inverse RoPE + FP8 output.
|
||||
|
||||
DSv4 always has 8 heads per ``wo_a`` group at any TP size, and the rest of
|
||||
the kernel's fixed shape (head/RoPE dims, per-128 einsum scales, FP32 RoPE
|
||||
cache) is what the SM100 layer uses anyway. What varies is the local head
|
||||
count and the configs the layer also serves unquantized.
|
||||
"""
|
||||
if layer.n_local_heads != _ROPE_QUANT_NUM_HEADS:
|
||||
# Padding fewer heads up to the kernel's fixed 128 costs more
|
||||
# attention than the fused epilogue saves.
|
||||
return f"RopeQuant needs {_ROPE_QUANT_NUM_HEADS} local query heads"
|
||||
if layer.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return "RopeQuant needs the per-tensor FP8 KV cache"
|
||||
if layer.wo_a.weight.dtype != torch.float8_e4m3fn:
|
||||
return "RopeQuant needs an FP8 wo_a"
|
||||
return None
|
||||
|
||||
|
||||
def rope_quant_attn_out(num_tokens: int, device: torch.device) -> QuantizedActivation:
|
||||
"""Allocate the output pair the RopeQuant kernel writes.
|
||||
|
||||
Values are group-major, ``[num_tokens, 16, 8 * 512]`` over physical
|
||||
``[group, token, K]``. Scales are MN-major packed UE8M0, one int32 per
|
||||
head over its four 128-wide blocks. Both are what ``fp8_einsum`` consumes.
|
||||
"""
|
||||
n_groups = _ROPE_QUANT_NUM_HEADS // _ROPE_QUANT_HEADS_PER_GROUP
|
||||
group_width = _ROPE_QUANT_HEADS_PER_GROUP * _ROPE_QUANT_HEAD_DIM
|
||||
data = torch.empty(
|
||||
(n_groups, num_tokens, group_width), dtype=torch.float8_e4m3fn, device=device
|
||||
).transpose(0, 1)
|
||||
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
|
||||
scale = torch.empty(
|
||||
(n_groups, group_width // (_ROPE_QUANT_BLOCK * 4), aligned),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).permute(2, 0, 1)[:num_tokens]
|
||||
return QuantizedActivation(
|
||||
data=data,
|
||||
scale=scale,
|
||||
orig_dtype=torch.bfloat16,
|
||||
orig_shape=data.shape,
|
||||
quant_key=kFp8Dynamic128Sym,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,21 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility imports for the platform-neutral DeepSeek-V4 vision model."""
|
||||
|
||||
from ..common.vl_model import (
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
from ..common.vl_model import (
|
||||
_make_deepseek_v4_vl_weights_mapper as _make_common_vl_weights_mapper,
|
||||
)
|
||||
from .model import _make_deepseek_v4_weights_mapper
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(expert_dtype: str, image_enabled: bool):
|
||||
"""Retain the original NVIDIA helper signature for downstream imports."""
|
||||
return _make_common_vl_weights_mapper(
|
||||
_make_deepseek_v4_weights_mapper(expert_dtype), image_enabled
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["DeepseekV4ForConditionalGeneration"]
|
||||
@@ -1,229 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Quantization config for DeepSeek V4."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
|
||||
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
|
||||
|
||||
def align_mtp_quant_config(
|
||||
quant_config: QuantizationConfig | None, config: PreTrainedConfig
|
||||
) -> None:
|
||||
"""Re-key the quant config's MTP entries onto the draft module prefixes.
|
||||
|
||||
``_make_deepseek_v4_weights_mapper`` renames the checkpoint's ``mtp.{i}.``
|
||||
layers to ``model.mtp.{i}.``, but the draft modules are built under
|
||||
``model.layers.{num_hidden_layers + i}.``. Without this remap no per-layer
|
||||
entry matches, so every MTP linear falls back to the global scheme. On a
|
||||
mixed checkpoint (FP8 attention and shared experts, NVFP4 routed experts)
|
||||
that builds NVFP4 parameters for FP8 weights and the load then fails a
|
||||
shape assertion.
|
||||
"""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
if quant_config is None:
|
||||
return
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
f"model.mtp.{i}.": f"model.layers.{config.num_hidden_layers + i}."
|
||||
for i in range(config.num_nextn_predict_layers)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FP8Config(Fp8Config):
|
||||
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
|
||||
|
||||
DeepSeek V4 checkpoints always use FP8 block quantization for
|
||||
linear/attention layers. The MoE expert weights vary by checkpoint:
|
||||
- ``expert_dtype="fp4"`` (e.g. DeepSeek-V4-Flash): MXFP4 experts
|
||||
with ue8m0 (e8m0fnu) FP8 linear scales.
|
||||
- ``expert_dtype="fp8"`` (e.g. DeepSeek-V4-Flash-Base): FP8 block
|
||||
experts with float32 FP8 linear scales.
|
||||
|
||||
The dispatch and the linear scale dtype are both keyed off
|
||||
``expert_dtype`` from the model's hf_config; missing values default
|
||||
to ``"fp4"`` so existing FP4 checkpoints stay unchanged.
|
||||
|
||||
NOTE: ``expert_dtype`` is resolved lazily because this config is
|
||||
constructed during VllmConfig setup, before ``set_current_vllm_config``
|
||||
is active. Reading hf_config eagerly in ``__init__`` would always see
|
||||
the default ``"fp4"`` and silently misroute Flash-Base checkpoints.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._resolved_expert_dtype: str | None = None
|
||||
self._resolved_moe_quant_algo: str | None = None
|
||||
self._nvfp4_config: ModelOptNvFp4Config | None = None
|
||||
# ``is_scale_e8m0`` is a property that resolves on first read,
|
||||
# by which time the current vllm_config has been set.
|
||||
|
||||
@property
|
||||
def expert_dtype(self) -> str:
|
||||
if self._resolved_expert_dtype is None:
|
||||
try:
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
except Exception:
|
||||
# vllm_config not yet set; defer the decision until a
|
||||
# later call lands inside set_current_vllm_config.
|
||||
return "fp4"
|
||||
expert_dtype = getattr(hf_config, "expert_dtype", "fp4")
|
||||
if expert_dtype not in _DEEPSEEK_V4_EXPERT_DTYPES:
|
||||
raise ValueError(
|
||||
f"Unsupported DeepSeek V4 expert_dtype={expert_dtype!r}; "
|
||||
f"expected one of {_DEEPSEEK_V4_EXPERT_DTYPES}."
|
||||
)
|
||||
self._resolved_expert_dtype = expert_dtype
|
||||
from vllm.logger import init_logger
|
||||
|
||||
init_logger(__name__).info_once(
|
||||
"DeepSeek V4 expert_dtype resolved to %r", expert_dtype
|
||||
)
|
||||
return self._resolved_expert_dtype
|
||||
|
||||
@property
|
||||
def is_scale_e8m0(self) -> bool:
|
||||
# FP4 checkpoints store FP8 linear scales as e8m0fnu; FP8 expert
|
||||
# checkpoints (Flash-Base) store them as float32.
|
||||
return self.expert_dtype == "fp4"
|
||||
|
||||
def _resolve_moe_overrides(self) -> None:
|
||||
if self._resolved_moe_quant_algo is not None:
|
||||
return
|
||||
try:
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
except Exception:
|
||||
return
|
||||
quant_cfg = getattr(hf_config, "quantization_config", None) or {}
|
||||
algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None
|
||||
self._resolved_moe_quant_algo = algo or ""
|
||||
|
||||
@property
|
||||
def moe_quant_algo(self) -> str:
|
||||
self._resolve_moe_overrides()
|
||||
return self._resolved_moe_quant_algo or ""
|
||||
|
||||
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
|
||||
if self._nvfp4_config is None:
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
|
||||
self._nvfp4_config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
group_size=16,
|
||||
quantization_args=(
|
||||
self.online_quantization_config.args
|
||||
if self.online_quantization_config is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return self._nvfp4_config
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "deepseek_v4_fp8"
|
||||
|
||||
@staticmethod
|
||||
def _is_quark_mxfp4_ocp(hf_quant_cfg: dict) -> bool:
|
||||
"""True for AMD-Quark exports whose global scheme is MXFP4."""
|
||||
weight = (hf_quant_cfg.get("global_quant_config") or {}).get("weight")
|
||||
# A non-dict weight (e.g. a list of multiple specs) means not an OCP
|
||||
# MXFP4 scheme (e.g. NVFP4 with 2-level scale).
|
||||
if not isinstance(weight, dict):
|
||||
return False
|
||||
return (
|
||||
weight.get("dtype") == "fp4"
|
||||
and weight.get("qscheme") == "per_group"
|
||||
and weight.get("group_size") == 32
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
# Quark checkpoints are always handled by QuarkConfig.
|
||||
if (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and hf_quant_cfg.get("quant_method") == "quark"
|
||||
):
|
||||
return None
|
||||
if not (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
):
|
||||
return None
|
||||
model_type = getattr(hf_config, "model_type", None)
|
||||
if model_type == "deepseek_v4" or user_quant == "deepseek_v4_fp8":
|
||||
return "deepseek_v4_fp8"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
|
||||
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
|
||||
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
|
||||
# so translate the schema into format Fp8Config.from_config expects.
|
||||
if config.get("quant_method") == "quark":
|
||||
quark_exclude = config.get("exclude") or []
|
||||
config = {
|
||||
"quant_method": "fp8",
|
||||
"activation_scheme": "dynamic",
|
||||
"fmt": "e4m3",
|
||||
"scale_fmt": "ue8m0",
|
||||
"weight_block_size": [128, 128],
|
||||
"ignored_layers": [
|
||||
name for name in quark_exclude if isinstance(name, str)
|
||||
],
|
||||
}
|
||||
return cast("DeepseekV4FP8Config", super().from_config(config))
|
||||
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.expert_dtype == "fp4":
|
||||
if self.moe_quant_algo == "NVFP4":
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
|
||||
return ModelOptNvFp4FusedMoE(
|
||||
quant_config=self._get_nvfp4_config(),
|
||||
moe_config=layer.moe_config,
|
||||
)
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
# expert_dtype == "fp8": fall through to Fp8Config which
|
||||
# returns Fp8MoEMethod with block-wise float32 scales.
|
||||
return super().get_quant_method(layer, prefix)
|
||||
@@ -1,524 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 FlashMLA sparse backend, metadata, and metadata builder."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
# Pad C128A topk width to this alignment. 128 covers both h_q=64 (B_TOPK=64) and
|
||||
# h_q=128 (B_TOPK=128). FlashMLA decode asserts extra_topk % B_TOPK == 0;
|
||||
# unaligned widths (e.g. 17 = ceil(2136/128)) crash the sm100 head64 kernel.
|
||||
# Padded slots stay -1 and decode_lens caps them via topk_length, so the pad is a
|
||||
# no-op at kernel level. Mirrors _SPARSE_PREFILL_TOPK_ALIGNMENT in cache_utils.py.
|
||||
_C128A_TOPK_ALIGNMENT = 128
|
||||
|
||||
|
||||
class DeepseekV4SparseMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4 sparse-MLA backend base.
|
||||
|
||||
Subclasses ``AttentionBackend`` directly (not the V3.2
|
||||
``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer
|
||||
(``DeepseekV4Attention``), so it does not reuse the V3.2 builder or impl, and
|
||||
only needs to declare its own metadata builder, KV-cache layout, and the
|
||||
sparse-MLA capability flags.
|
||||
"""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"fp8_ds_mla",
|
||||
"fp8", # alias for fp8_ds_mla
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [256]
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]:
|
||||
return DeepseekV4SparseMLAMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[Any]:
|
||||
# DeepSeek-V4 runs its attention through ``DeepseekV4Attention.forward``,
|
||||
# not the generic ``Attention``/``MLAAttention`` layer, so the backend's
|
||||
# impl class is never instantiated.
|
||||
raise NotImplementedError(
|
||||
"DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 "
|
||||
"attention runs through DeepseekV4Attention."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
# DeepSeek V4 layout: 448 NoPE + 64 RoPE = 512.
|
||||
return [512]
|
||||
|
||||
@classmethod
|
||||
def is_mla(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major in [9, 10]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV4FlashMLAMetadata(AttentionMetadata):
|
||||
num_reqs: int
|
||||
max_query_len: int
|
||||
max_seq_len: int
|
||||
|
||||
num_actual_tokens: int # Number of tokens excluding padding.
|
||||
query_start_loc: torch.Tensor
|
||||
slot_mapping: torch.Tensor
|
||||
|
||||
block_table: torch.Tensor
|
||||
req_id_per_token: torch.Tensor
|
||||
block_size: int
|
||||
topk_tokens: int
|
||||
|
||||
# Pre-computed C128A metadata (compress_ratio == 128 only).
|
||||
# Decode: global slot ids + valid-entry counts (fused from positions).
|
||||
c128a_global_decode_topk_indices: torch.Tensor | None = None
|
||||
c128a_decode_topk_lens: torch.Tensor | None = None
|
||||
# Prefill: local topk indices (used by combine_topk_swa_indices).
|
||||
c128a_prefill_topk_indices: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV4SparseMLAMetadataBuilder(
|
||||
AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata]
|
||||
):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.model_config = vllm_config.model_config
|
||||
# Classify single-token queries (plus num_speculative_tokens via
|
||||
# supports_spec_as_decode=True) as decodes; longer queries go to prefill.
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
self.topk_tokens = self.model_config.hf_config.index_topk
|
||||
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.req_id_per_token_buffer = torch.empty(
|
||||
(max_num_batched_tokens,), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
|
||||
self.compress_ratio = self.kv_cache_spec.tokens_per_state
|
||||
|
||||
# Pre-allocate compressed slot mapping buffer for CUDA graph address
|
||||
# stability when compress_ratio > 1.
|
||||
if self.compress_ratio > 1:
|
||||
self.compressed_slot_mapping_buffer = torch.empty(
|
||||
max_num_batched_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Pre-allocate C128A topk buffers for CUDA graph address stability.
|
||||
if self.compress_ratio == 128:
|
||||
c128a_max_compressed = cdiv(
|
||||
self.model_config.max_model_len, self.compress_ratio
|
||||
)
|
||||
c128a_max_compressed = (
|
||||
cdiv(c128a_max_compressed, _C128A_TOPK_ALIGNMENT)
|
||||
* _C128A_TOPK_ALIGNMENT
|
||||
)
|
||||
# Stored so _build_c128a_metadata passes it as the kernel's
|
||||
# max_compressed_tokens, matching the buffer stride. Otherwise the
|
||||
# kernel's default 8192 iterates past row width and spills writes
|
||||
# into adjacent rows (present in both decode and prefill branches of
|
||||
# _build_c128a_topk_metadata_kernel).
|
||||
self.c128a_max_compressed = c128a_max_compressed
|
||||
self.c128a_global_decode_buffer = torch.empty(
|
||||
(max_num_batched_tokens, c128a_max_compressed),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.c128a_decode_lens_buffer = torch.empty(
|
||||
max_num_batched_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
self.c128a_prefill_buffer = torch.empty(
|
||||
(max_num_batched_tokens, c128a_max_compressed),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> DeepseekV4FlashMLAMetadata:
|
||||
cm = common_attn_metadata
|
||||
req_id_per_token = cm.token_to_req_indices(self.req_id_per_token_buffer)
|
||||
|
||||
slot_mapping = cm.slot_mapping
|
||||
if self.compress_ratio > 1:
|
||||
slot_mapping = get_compressed_slot_mapping(
|
||||
cm.num_actual_tokens,
|
||||
cm.slot_mapping,
|
||||
cm.query_start_loc,
|
||||
cm.seq_lens,
|
||||
cm.block_table_tensor.clamp_(min=0),
|
||||
int(self.kv_cache_spec.num_states),
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
)
|
||||
|
||||
c128a_fields: dict[str, torch.Tensor | None] = {}
|
||||
if self.compress_ratio == 128:
|
||||
c128a_fields = self._build_c128a_metadata(cm, req_id_per_token)
|
||||
|
||||
return DeepseekV4FlashMLAMetadata(
|
||||
num_reqs=cm.num_reqs,
|
||||
max_query_len=cm.max_query_len,
|
||||
max_seq_len=cm.max_seq_len,
|
||||
num_actual_tokens=cm.num_actual_tokens,
|
||||
query_start_loc=cm.query_start_loc,
|
||||
slot_mapping=slot_mapping,
|
||||
block_table=cm.block_table_tensor,
|
||||
req_id_per_token=req_id_per_token,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
topk_tokens=self.topk_tokens,
|
||||
c128a_global_decode_topk_indices=c128a_fields.get(
|
||||
"c128a_global_decode_topk_indices"
|
||||
),
|
||||
c128a_decode_topk_lens=c128a_fields.get("c128a_decode_topk_lens"),
|
||||
c128a_prefill_topk_indices=c128a_fields.get("c128a_prefill_topk_indices"),
|
||||
)
|
||||
|
||||
def _build_c128a_metadata(
|
||||
self,
|
||||
cm: CommonAttentionMetadata,
|
||||
req_id_per_token: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
"""Pre-compute C128A topk indices for DeepseekV4 (compress_ratio >= 128)."""
|
||||
# Must match SWA's decode split (no `require_uniform=True`) so
|
||||
# `c128a_global_decode_topk_indices.shape[0]` lines up with q in
|
||||
# `_forward_decode`. The per-token C128A kernel handles non-uniform
|
||||
# query lengths.
|
||||
(num_decodes, _, num_decode_tokens, num_prefill_tokens) = (
|
||||
split_decodes_and_prefills(
|
||||
cm,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
)
|
||||
)
|
||||
|
||||
num_total = num_decode_tokens + num_prefill_tokens
|
||||
if num_total == 0:
|
||||
return {}
|
||||
|
||||
assert cm.positions is not None, (
|
||||
"positions is required for C128A metadata build"
|
||||
)
|
||||
active_topk_width = min(
|
||||
max(
|
||||
triton.next_power_of_2(max(cm.max_seq_len // self.compress_ratio, 1)),
|
||||
_C128A_TOPK_ALIGNMENT,
|
||||
),
|
||||
self.c128a_max_compressed,
|
||||
)
|
||||
assert active_topk_width >= cm.max_seq_len // self.compress_ratio
|
||||
assert active_topk_width % _C128A_TOPK_ALIGNMENT == 0
|
||||
block_size = self.kv_cache_spec.block_size // self.compress_ratio
|
||||
global_decode, decode_lens, prefill_local = build_c128a_topk_metadata(
|
||||
cm.positions[:num_total],
|
||||
self.compress_ratio,
|
||||
num_decode_tokens,
|
||||
req_id_per_token,
|
||||
cm.block_table_tensor[:num_decodes],
|
||||
block_size,
|
||||
cm.slot_mapping,
|
||||
self.c128a_global_decode_buffer,
|
||||
self.c128a_decode_lens_buffer,
|
||||
self.c128a_prefill_buffer,
|
||||
max_compressed_tokens=active_topk_width,
|
||||
)
|
||||
|
||||
result: dict[str, torch.Tensor | None] = {}
|
||||
if num_decode_tokens > 0:
|
||||
result["c128a_global_decode_topk_indices"] = global_decode.view(
|
||||
num_decode_tokens, 1, -1
|
||||
)
|
||||
result["c128a_decode_topk_lens"] = decode_lens
|
||||
if num_prefill_tokens > 0:
|
||||
result["c128a_prefill_topk_indices"] = prefill_local
|
||||
return result
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
|
||||
|
||||
def build_c128a_topk_metadata(
|
||||
positions: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
num_decode_tokens: int,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
global_decode_buffer: torch.Tensor,
|
||||
decode_lens_buffer: torch.Tensor,
|
||||
prefill_buffer: torch.Tensor,
|
||||
max_compressed_tokens: int = 8192,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Single kernel for all C128A tokens (decode + prefill).
|
||||
|
||||
Decode tokens: position → block_table lookup → global slot ids + topk_lens.
|
||||
Prefill tokens: position → local indices [0, ..., n-1, -1, ...].
|
||||
|
||||
Writes into pre-allocated buffers for CUDA graph address stability.
|
||||
Returns views of the buffers.
|
||||
"""
|
||||
num_tokens = positions.shape[0]
|
||||
num_prefill_tokens = num_tokens - num_decode_tokens
|
||||
assert max_compressed_tokens % _C128A_TOPK_ALIGNMENT == 0
|
||||
assert (
|
||||
0
|
||||
< max_compressed_tokens
|
||||
<= min(global_decode_buffer.shape[1], prefill_buffer.shape[1])
|
||||
)
|
||||
assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1
|
||||
|
||||
# TODO: support adaptive-width decode on SM120 (needs the FlashInfer
|
||||
# SM120 kernel to accept a real row stride for eidx).
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is not None and capability.major == 12:
|
||||
global_decode = global_decode_buffer[:num_decode_tokens]
|
||||
else:
|
||||
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
|
||||
decode_lens = decode_lens_buffer[:num_decode_tokens]
|
||||
prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens]
|
||||
assert global_decode.stride(0) == global_decode_buffer.stride(0)
|
||||
assert prefill_local.stride(0) == prefill_buffer.stride(0)
|
||||
|
||||
if num_tokens == 0:
|
||||
return global_decode, decode_lens, prefill_local
|
||||
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL(
|
||||
global_decode_buffer,
|
||||
decode_lens_buffer,
|
||||
prefill_buffer,
|
||||
positions,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
num_decode_tokens,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_size,
|
||||
slot_mapping,
|
||||
)
|
||||
return global_decode, decode_lens, prefill_local
|
||||
|
||||
|
||||
class BuildC128ATopkMetadataKernel(
|
||||
VllmTritonJitKernel["BuildC128ATopkMetadataKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
compress_ratio: int
|
||||
max_compressed_tokens: int
|
||||
block_size: int
|
||||
triton_block_size: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"global_decode_stride",
|
||||
"prefill_local_stride",
|
||||
"num_decode_tokens",
|
||||
"block_table_stride",
|
||||
]
|
||||
)
|
||||
def kernel(
|
||||
# Decode outputs
|
||||
global_decode_ptr,
|
||||
global_decode_stride,
|
||||
decode_lens_ptr,
|
||||
# Prefill output
|
||||
prefill_local_ptr,
|
||||
prefill_local_stride,
|
||||
# Inputs
|
||||
positions_ptr,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
num_decode_tokens,
|
||||
token_to_req_indices_ptr,
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
slot_mapping_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
num_compressed = (position + 1) // compress_ratio
|
||||
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
|
||||
is_decode = token_idx < num_decode_tokens
|
||||
|
||||
if is_decode:
|
||||
# --- Decode: block-table lookup → global slot ids + count ---
|
||||
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
is_valid = offset < num_compressed
|
||||
|
||||
block_indices = offset // block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + block_indices,
|
||||
mask=mask & is_valid,
|
||||
)
|
||||
block_offsets = offset % block_size
|
||||
slot_ids = block_numbers * block_size + block_offsets
|
||||
slot_ids = tl.where(is_valid, slot_ids, -1)
|
||||
tl.store(
|
||||
global_decode_ptr + token_idx * global_decode_stride + offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
)
|
||||
count += tl.sum(is_valid.to(tl.int32), axis=0)
|
||||
|
||||
tl.store(
|
||||
decode_lens_ptr + token_idx,
|
||||
tl.where(is_valid_token, count, 0),
|
||||
)
|
||||
else:
|
||||
# --- Prefill: write local indices ---
|
||||
pfx_idx = token_idx - num_decode_tokens
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
tl.store(
|
||||
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
|
||||
tl.where(offset < num_compressed, offset, -1),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if getattr(hf_config, "model_type", None) != "deepseek_v4":
|
||||
return []
|
||||
|
||||
compress_ratio = 128
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
if max_model_len <= 0:
|
||||
return []
|
||||
max_compressed_tokens = cdiv(max_model_len, compress_ratio)
|
||||
max_compressed_tokens = (
|
||||
cdiv(max_compressed_tokens, _C128A_TOPK_ALIGNMENT) * _C128A_TOPK_ALIGNMENT
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
compress_ratio=compress_ratio,
|
||||
max_compressed_tokens=max_compressed_tokens,
|
||||
# DeepSeek V4 sparse MLA uses 256-token KV pages; C128A metadata
|
||||
# works in compressed-token units.
|
||||
block_size=256 // compress_ratio,
|
||||
triton_block_size=1024,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
global_decode_buffer=TritonWarmupTensor(
|
||||
torch.int32, shape=(1, 1), strides=(1, 1)
|
||||
),
|
||||
decode_lens_buffer=int32_ptr,
|
||||
prefill_buffer=TritonWarmupTensor(
|
||||
torch.int32, shape=(1, 1), strides=(1, 1)
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
max_compressed_tokens=compile_key.max_compressed_tokens,
|
||||
num_decode_tokens=0,
|
||||
token_to_req_indices=int32_ptr,
|
||||
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
|
||||
block_size=compile_key.block_size,
|
||||
slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
global_decode_buffer: torch.Tensor,
|
||||
decode_lens_buffer: torch.Tensor,
|
||||
prefill_buffer: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
max_compressed_tokens: int,
|
||||
num_decode_tokens: int,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> LaunchSpec:
|
||||
return (positions.shape[0],), dict(
|
||||
global_decode_ptr=global_decode_buffer,
|
||||
global_decode_stride=global_decode_buffer.stride(0),
|
||||
decode_lens_ptr=decode_lens_buffer,
|
||||
prefill_local_ptr=prefill_buffer,
|
||||
prefill_local_stride=prefill_buffer.stride(0),
|
||||
block_table_stride=block_table.stride(0),
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL = BuildC128ATopkMetadataKernel()
|
||||
@@ -1,14 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stub for platforms where the DeepSeek-V4 vision variant is unsupported."""
|
||||
|
||||
from torch import nn
|
||||
|
||||
|
||||
class DeepseekV4ForConditionalGeneration(nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix: str = ""):
|
||||
super().__init__()
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 vision (DeepseekV4ForConditionalGeneration) is not "
|
||||
"supported on the current platform; use CUDA or ROCm."
|
||||
)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,419 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark draft model for DeepSeek-V4 on Intel XPU.
|
||||
|
||||
Minimal XPU port of nvidia/dspark.py. Replaces tilelang MHC kernels with
|
||||
the platform-agnostic custom ops (HCHeadOp, MHCPostOp) already used by the
|
||||
XPU MTP path, and uses the XPU Triton-based qnorm_rope_kv_fp8_insert for
|
||||
context KV precomputation.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp, MHCPostOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DSparkDeepseekV4Model(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_eps = config.hc_eps
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
|
||||
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
|
||||
|
||||
# Shared with target (aliased by speculator loading utility).
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
self.main_proj = ReplicatedLinear(
|
||||
config.hidden_size * len(self.target_layer_ids),
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "main_proj"),
|
||||
)
|
||||
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4DecoderLayer(
|
||||
current_vllm_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
|
||||
)
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov head
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
|
||||
# XPU MHC ops (replaces tilelang)
|
||||
self.mhc_post_op = MHCPostOp()
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""main_x = main_norm(main_proj(concat of target aux hidden states))."""
|
||||
return self.main_norm(self.main_proj(aux_hidden_states))
|
||||
|
||||
@torch.inference_mode()
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
main_x: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Insert the sliding-window context KV for every draft layer.
|
||||
|
||||
Each layer derives its context KV from the SAME projected target hidden
|
||||
``main_x``, via that layer's own wkv + kv_norm + RoPE + quant, then
|
||||
writes it at the layer's context slots.
|
||||
"""
|
||||
for i, layer in enumerate(self.layers):
|
||||
slot_mapping = (
|
||||
None if context_slot_mappings is None else context_slot_mappings[i]
|
||||
)
|
||||
attn = layer.attn
|
||||
# wkv part of the fused wq_a|wkv projection (q_lora part discarded)
|
||||
qr_kv, _ = attn.fused_wqa_wkv(main_x)
|
||||
kv = qr_kv[..., attn.q_lora_rank :]
|
||||
kv = attn.kv_norm(kv)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
_insert_context_kv(attn, kv, context_positions, slot_mapping)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
|
||||
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
|
||||
residual = post_mix = res_mix = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
# mhc_post: merge hyper-connection copies
|
||||
hidden_states = self.mhc_post_op(hidden_states, residual, post_mix, res_mix)
|
||||
# hc_head: reduces hc copies; return pre-norm head hidden
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
self.hc_head_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _insert_context_kv(
|
||||
attn: nn.Module,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
|
||||
|
||||
On XPU, we reuse the same xpu_qnorm_rope_kv_fp8_insert kernel used in
|
||||
the forward path, passing a dummy q (result discarded).
|
||||
"""
|
||||
from vllm.models.deepseek_v4.xpu.xpu_qnorm_rope_kv_fp8_insert import (
|
||||
xpu_qnorm_rope_kv_fp8_insert,
|
||||
)
|
||||
|
||||
swa_cache = attn.swa_cache_layer.kv_cache
|
||||
block_size = attn.swa_cache_layer.block_size
|
||||
cos_sin_cache = attn.rotary_emb.cos_sin_cache
|
||||
n_ctx = kv.shape[0]
|
||||
|
||||
# Dummy q — we only care about the KV insert side effect.
|
||||
dummy_q = torch.empty(
|
||||
(n_ctx, attn.n_local_heads, attn.head_dim),
|
||||
dtype=kv.dtype,
|
||||
device=kv.device,
|
||||
)
|
||||
xpu_qnorm_rope_kv_fp8_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
"""XPU DSpark draft model entry point for DeepSeek-V4."""
|
||||
|
||||
has_own_embed_tokens = False
|
||||
has_own_lm_head = False
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
self.model = DSparkDeepseekV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
# Shared with the target (aliased by the speculator's load utility).
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
# --- Hooks used by the speculator ---
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.combine_hidden_states(aux_hidden_states)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
|
||||
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
self.model.precompute_and_store_context_kv(
|
||||
context_states, context_positions, context_slot_mappings
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Base logits U_k = lm_head(norm(head_hidden))."""
|
||||
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids # full-vocab: draft ids are target ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
# --- Weight loading ---
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load ``mtp.{0,1,2}.*`` draft weights from the target checkpoint."""
|
||||
first_layer = self.model.layers[0]
|
||||
use_mega_moe = first_layer.ffn.use_mega_moe
|
||||
if use_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
else:
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
|
||||
# .scale -> per-method scale suffix
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
|
||||
# Expert weights
|
||||
if ".experts." in name:
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if name_mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
success = param.weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
|
||||
# Stacked params (decoder-layer only)
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
break
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
self._finalize_moe()
|
||||
logger.info_once("DSpark XPU draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def _finalize_moe(self) -> None:
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map checkpoint ``mtp.{i}.*`` name to this model's parameter path."""
|
||||
m = re.match(r"mtp\.(\d+)\.(.*)", name)
|
||||
if m is None:
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
if rest.startswith("confidence_head."):
|
||||
return None
|
||||
head_prefixes = (
|
||||
"norm.",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
"markov_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
):
|
||||
return f"model.{rest}"
|
||||
return f"model.layers.{stage}.{rest}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,510 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
|
||||
|
||||
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
|
||||
pieces that have no analogue in V3/V32:
|
||||
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
|
||||
the fused ``eh_proj``);
|
||||
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
|
||||
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
|
||||
* V4-specific checkpoint weight-name remapping in ``load_weights``.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
# Other FP8 linear scales (including shared experts) always use
|
||||
# ``.weight_scale_inv``. Mirrors the per-instance mapper built by
|
||||
# ``_make_deepseek_v4_weights_mapper`` in deepseek_v4.py.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
prefix: str,
|
||||
aux_stream_list: list | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
|
||||
# fusing them the way V3 does with eh_proj.
|
||||
self.e_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.e_proj",
|
||||
)
|
||||
self.h_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.h_proj",
|
||||
)
|
||||
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.mtp_block = DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
|
||||
# reshape to (T, hc_mult, D) — the training-time layout — before
|
||||
# the fused norm pass so both inputs are 3D-friendly.
|
||||
previous_hidden_states = previous_hidden_states.view(
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight.data,
|
||||
self.hnorm.weight.data,
|
||||
self.enorm.variance_epsilon,
|
||||
self.hc_mult,
|
||||
)
|
||||
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
)
|
||||
hidden_states = self.mtp_block.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
# Return the flat pre-hc_head residual so it can be re-fed as the
|
||||
# next spec step's `previous_hidden_states` when
|
||||
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
|
||||
return hidden_states.flatten(1)
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.device = current_platform.device_type
|
||||
|
||||
topk_tokens = config.index_topk
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
# Disable aux streams on XPU — no multi-stream overlap support.
|
||||
aux_stream_list = None
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepSeekV4MultiTokenPredictorLayer(
|
||||
vllm_config,
|
||||
self.topk_indices_buffer,
|
||||
f"{prefix}.layers.{idx}",
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
|
||||
# hc_head here so logits are computed from the dense hidden state.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
|
||||
)
|
||||
hidden_states = mtp_layer.hc_head_op(
|
||||
hidden_states,
|
||||
mtp_layer.hc_head_fn,
|
||||
mtp_layer.hc_head_scale,
|
||||
mtp_layer.hc_head_base,
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
)
|
||||
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
return logits
|
||||
|
||||
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
WEIGHT_NAME_REMAPPING: dict[str, str] = {
|
||||
".emb.tok_emb.weight": ".embed_tokens.weight",
|
||||
".head.weight": ".shared_head.head.weight",
|
||||
".norm.weight": ".shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def _remap_weight_name(name: str) -> str:
|
||||
"""Remap checkpoint weight names to model parameter names."""
|
||||
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
|
||||
if old_pattern in name:
|
||||
name = name.replace(old_pattern, new_pattern)
|
||||
return name
|
||||
|
||||
def _find_mtp_layer_idx(name: str) -> int:
|
||||
subnames = name.split(".")
|
||||
for subname in subnames:
|
||||
try:
|
||||
# we return the first encountered integer
|
||||
return int(subname)
|
||||
except ValueError:
|
||||
continue
|
||||
return 0
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
else:
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
|
||||
# FP4/MXFP4 experts register ``..._weight_scale``. Choose the suffix
|
||||
# for the rename below based on the model's expert dtype.
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer_idx = _find_mtp_layer_idx(name)
|
||||
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
|
||||
# `model.layers.{num_hidden_layers + i}.*` so that
|
||||
# get_spec_layer_idx_from_weight_name can identify them.
|
||||
name = name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
name = _remap_weight_name(name)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
|
||||
continue
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# Reinterpret E8M0 scales as uint8 to preserve raw
|
||||
# exponent bytes; numeric copy_() would zero them.
|
||||
# Mirrors the main DeepseekV4 loader.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
# ``e_score_correction_bias`` lives on the gate
|
||||
# under a different attribute name.
|
||||
name = name.replace(
|
||||
".ffn.gate.bias",
|
||||
".ffn.gate.e_score_correction_bias",
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled():
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may have "
|
||||
f"been quantized without including the MTP layers. "
|
||||
f"Use a checkpoint that includes MTP layer weights, "
|
||||
f"or disable speculative decoding."
|
||||
)
|
||||
self.finalize_mega_moe_weights()
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for layer in self.model.layers.values():
|
||||
layer.mtp_block.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"h_proj",
|
||||
"e_proj",
|
||||
"shared_head",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
# treat rest weights as weights for transformer layer block
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
# treat shared weights as top level weights
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""XPU Triton replacement for fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert.
|
||||
|
||||
Does: Q per-head RMSNorm + GPT-J RoPE, KV GPT-J RoPE + UE8M0 FP8 quant + insert.
|
||||
Uses the existing quantize_and_insert_k_cache for the FP8 portion.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
HEAD_DIM = 512
|
||||
ROPE_DIM = 64
|
||||
NOPE_DIM = HEAD_DIM - ROPE_DIM
|
||||
HALF_ROPE = ROPE_DIM // 2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _xpu_qnorm_rope_kernel(
|
||||
q_ptr, # [num_tokens, num_heads, HEAD_DIM]
|
||||
kv_ptr, # [num_tokens, HEAD_DIM]
|
||||
kv_out_ptr, # [num_tokens, HEAD_DIM] bf16 (RoPE-applied kv for cache insert)
|
||||
position_ids_ptr,
|
||||
cos_sin_cache_ptr,
|
||||
eps: tl.constexpr,
|
||||
num_tokens,
|
||||
num_heads: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
ROPE_DIM: tl.constexpr,
|
||||
NOPE_DIM: tl.constexpr,
|
||||
HALF_ROPE: tl.constexpr,
|
||||
):
|
||||
"""Apply per-head RMSNorm + GPT-J RoPE on Q, GPT-J RoPE on KV.
|
||||
|
||||
GPT-J interleaved format: pairs are (data[2i], data[2i+1]).
|
||||
cos_sin_cache layout: [max_pos, ROPE_DIM] with first HALF_ROPE=cos,
|
||||
second HALF_ROPE=sin.
|
||||
"""
|
||||
token_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
if token_idx >= num_tokens:
|
||||
return
|
||||
|
||||
pos = tl.load(position_ids_ptr + token_idx).to(tl.int64)
|
||||
|
||||
# Load cos/sin for this position
|
||||
rope_pair_idx = tl.arange(0, HALF_ROPE)
|
||||
cos_val = tl.load(cos_sin_cache_ptr + pos * ROPE_DIM + rope_pair_idx).to(tl.float32)
|
||||
sin_val = tl.load(
|
||||
cos_sin_cache_ptr + pos * ROPE_DIM + HALF_ROPE + rope_pair_idx
|
||||
).to(tl.float32)
|
||||
|
||||
if head_idx < num_heads:
|
||||
# ========== Q: per-head RMSNorm + GPT-J RoPE ==========
|
||||
q_base = q_ptr + token_idx * num_heads * HEAD_DIM + head_idx * HEAD_DIM
|
||||
|
||||
# Load full head
|
||||
offs = tl.arange(0, HEAD_DIM)
|
||||
q_vals = tl.load(q_base + offs).to(tl.float32)
|
||||
|
||||
# RMSNorm (no weight)
|
||||
sq_sum = tl.sum(q_vals * q_vals, axis=0)
|
||||
rms = tl.rsqrt(sq_sum / HEAD_DIM + eps)
|
||||
q_vals = q_vals * rms
|
||||
|
||||
# Store ONLY the NoPE portion (positions 0..NOPE_DIM-1)
|
||||
nope_mask = offs < NOPE_DIM
|
||||
tl.store(q_base + offs, q_vals.to(q_ptr.type.element_ty), mask=nope_mask)
|
||||
|
||||
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions:
|
||||
even_offs = NOPE_DIM + rope_pair_idx * 2
|
||||
odd_offs = NOPE_DIM + rope_pair_idx * 2 + 1
|
||||
|
||||
# Re-load original values at rope positions and normalize
|
||||
q_even = tl.load(q_base + even_offs).to(tl.float32) * rms
|
||||
q_odd = tl.load(q_base + odd_offs).to(tl.float32) * rms
|
||||
|
||||
new_even = q_even * cos_val - q_odd * sin_val
|
||||
new_odd = q_even * sin_val + q_odd * cos_val
|
||||
|
||||
# Store rotated RoPE values
|
||||
tl.store(q_base + even_offs, new_even.to(q_ptr.type.element_ty))
|
||||
tl.store(q_base + odd_offs, new_odd.to(q_ptr.type.element_ty))
|
||||
else:
|
||||
# ========== KV: GPT-J RoPE only ==========
|
||||
kv_base = kv_ptr + token_idx * HEAD_DIM
|
||||
kv_out_base = kv_out_ptr + token_idx * HEAD_DIM
|
||||
|
||||
# Copy only the NoPE portion; the RoPE region is written exclusively
|
||||
# by the rotated stores below. Masking avoids a write-after-write
|
||||
# hazard on the overlapping RoPE addresses (matches the Q branch).
|
||||
offs = tl.arange(0, HEAD_DIM)
|
||||
kv_full = tl.load(kv_base + offs)
|
||||
nope_mask = offs < NOPE_DIM
|
||||
tl.store(kv_out_base + offs, kv_full, mask=nope_mask)
|
||||
|
||||
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions
|
||||
even_offs = NOPE_DIM + rope_pair_idx * 2
|
||||
odd_offs = NOPE_DIM + rope_pair_idx * 2 + 1
|
||||
|
||||
kv_even = tl.load(kv_base + even_offs).to(tl.float32)
|
||||
kv_odd = tl.load(kv_base + odd_offs).to(tl.float32)
|
||||
|
||||
new_even = kv_even * cos_val - kv_odd * sin_val
|
||||
new_odd = kv_even * sin_val + kv_odd * cos_val
|
||||
|
||||
tl.store(kv_out_base + even_offs, new_even.to(kv_out_ptr.type.element_ty))
|
||||
tl.store(kv_out_base + odd_offs, new_odd.to(kv_out_ptr.type.element_ty))
|
||||
|
||||
|
||||
def xpu_qnorm_rope_kv_fp8_insert(
|
||||
q: torch.Tensor, # [num_tokens, num_heads, HEAD_DIM] bf16, in-place
|
||||
kv: torch.Tensor, # [num_tokens, HEAD_DIM] bf16
|
||||
swa_kv_cache: torch.Tensor, # [num_blocks, block_size, 584] or flat uint8
|
||||
slot_mapping: torch.Tensor, # [num_tokens] int64
|
||||
positions: torch.Tensor, # [num_tokens] int64
|
||||
cos_sin_cache: torch.Tensor, # [max_pos, ROPE_DIM]
|
||||
eps: float,
|
||||
block_size: int,
|
||||
):
|
||||
"""XPU Triton: qnorm+rope on Q, rope on KV, then FP8 UE8M0 quant+insert."""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
quantize_and_insert_k_cache,
|
||||
)
|
||||
|
||||
num_tokens = q.shape[0]
|
||||
num_heads = q.shape[1]
|
||||
|
||||
# Allocate temp buffer for RoPE-applied KV
|
||||
kv_roped = torch.empty_like(kv)
|
||||
|
||||
# Grid: one program per (token, head_or_kv)
|
||||
# head_idx < num_heads: process Q head
|
||||
# head_idx == num_heads: process KV
|
||||
grid = (num_tokens, num_heads + 1)
|
||||
_xpu_qnorm_rope_kernel[grid](
|
||||
q,
|
||||
kv,
|
||||
kv_roped,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
num_tokens,
|
||||
num_heads=num_heads,
|
||||
HEAD_DIM=HEAD_DIM,
|
||||
ROPE_DIM=ROPE_DIM,
|
||||
NOPE_DIM=NOPE_DIM,
|
||||
HALF_ROPE=HALF_ROPE,
|
||||
)
|
||||
|
||||
# FP8 UE8M0 quant + paged insert (reuse existing Triton kernel)
|
||||
# swa_kv_cache may be [num_blocks, block_size, 584] or [num_blocks, flat]
|
||||
# quantize_and_insert_k_cache expects [num_blocks, block_bytes] uint8
|
||||
cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
quantize_and_insert_k_cache(
|
||||
kv_roped,
|
||||
cache_2d,
|
||||
slot_mapping,
|
||||
block_size=block_size,
|
||||
)
|
||||
@@ -1,368 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""XPU DeepSeek-V4 attention subclass.
|
||||
|
||||
Subclasses the shared ``DeepseekV4Attention`` ABC and provides XPU-native
|
||||
Triton kernels for decode (FP8 dequant + BF16 attention) and prefill
|
||||
(BF16 gathered KV + sparse attention).
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
)
|
||||
from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import (
|
||||
xpu_sparse_decode_fp8,
|
||||
)
|
||||
from vllm.v1.attention.ops.xpu_mla_sparse import triton_bf16_mla_sparse_interface
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "XPU_V4_MLA_SPARSE"
|
||||
|
||||
|
||||
class DeepseekV4XPUAttention(DeepseekV4Attention):
|
||||
"""XPU sparse MLA attention layer for DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4XPUSparseBackend
|
||||
use_flashmla_fp8_layout = True
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
# torch.cuda.Event() raises RuntimeError on XPU ("dummy base class").
|
||||
# The Base and DeepseekV4Indexer both create cuda Events in __init__, so
|
||||
# we temporarily redirect torch.cuda.Event → torch.xpu.Event.
|
||||
_orig_event = torch.cuda.Event
|
||||
torch.cuda.Event = torch.xpu.Event # type: ignore[misc]
|
||||
try:
|
||||
super().__init__(*args, **kwargs)
|
||||
finally:
|
||||
torch.cuda.Event = _orig_event # type: ignore[misc]
|
||||
|
||||
def _fused_qnorm_rope_kv_insert(self, q, kv, positions, attn_metadata):
|
||||
from typing import cast
|
||||
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: no-op, just return q (no padding needed on XPU).
|
||||
return q
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
from vllm.models.deepseek_v4.xpu.xpu_qnorm_rope_kv_fp8_insert import (
|
||||
xpu_qnorm_rope_kv_fp8_insert,
|
||||
)
|
||||
|
||||
xpu_qnorm_rope_kv_fp8_insert(
|
||||
q,
|
||||
kv,
|
||||
self.swa_cache_layer.kv_cache,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
return q
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return num_heads
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
|
||||
o_fp8, o_scale = fused_inv_rope_fp8_quant(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
tma_aligned_scales=False,
|
||||
)
|
||||
|
||||
# Precomputed contiguous [G, K, N] weight and [G, K/bs, N/bs] scale.
|
||||
wo_a_weight = self.wo_a.bmm_weight
|
||||
wo_a_scale = self.wo_a.bmm_scale
|
||||
|
||||
# TODO: optimize fused_inv_rope_fp8_quant for xpu bmm to
|
||||
# eliminate o_scale transpose + contiguous
|
||||
z = torch.ops.vllm.xpu_fp8_bmm(
|
||||
o_fp8.transpose(0, 1),
|
||||
wo_a_weight,
|
||||
torch.bfloat16,
|
||||
o_scale.transpose(0, 1).contiguous(),
|
||||
wo_a_scale,
|
||||
None,
|
||||
)
|
||||
|
||||
return self.wo_b(z.transpose(0, 1).flatten(1))
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert output.shape == q.shape, (
|
||||
f"output buffer shape {output.shape} must match q shape {q.shape}"
|
||||
)
|
||||
assert output.dtype == q.dtype, (
|
||||
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: reserve workspace, skip actual kernels.
|
||||
swa_only = self.compress_ratio <= 1
|
||||
N = (
|
||||
0
|
||||
if swa_only
|
||||
else (self.max_model_len + self.compress_ratio - 1)
|
||||
// self.compress_ratio
|
||||
)
|
||||
M = N + self.window_size + self.max_num_batched_tokens
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
|
||||
)
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
self_kv_cache = self.kv_cache if not swa_only else None
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
|
||||
# Split prefill and decode
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
if num_prefills > 0:
|
||||
self._forward_prefill(
|
||||
q=q[num_decode_tokens:],
|
||||
positions=positions[num_decode_tokens:],
|
||||
compressed_k_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
output=output[num_decode_tokens:],
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
)
|
||||
if num_decodes > 0:
|
||||
self._forward_decode(
|
||||
q=q[:num_decode_tokens],
|
||||
kv_cache=self_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
def _forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
topk_indices = None
|
||||
topk_lens = None
|
||||
if not swa_only:
|
||||
assert attn_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if self.compress_ratio == 4:
|
||||
# C4A: local indices differ per layer (filled by Indexer).
|
||||
assert self.topk_indices_buffer is not None
|
||||
global_indices, topk_lens = compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
)
|
||||
topk_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
else:
|
||||
# C128A: pre-computed during metadata build.
|
||||
topk_indices = attn_metadata.c128a_global_decode_topk_indices
|
||||
topk_lens = attn_metadata.c128a_decode_topk_lens
|
||||
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
|
||||
assert swa_indices is not None and swa_lens is not None
|
||||
xpu_sparse_decode_fp8(
|
||||
q=q,
|
||||
kv_cache=kv_cache,
|
||||
swa_kv_cache=self.swa_cache_layer.kv_cache,
|
||||
swa_only=swa_only,
|
||||
topk_indices=topk_indices,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_indices,
|
||||
swa_lens=swa_lens,
|
||||
attn_sink=self.attn_sink,
|
||||
softmax_scale=self.scale,
|
||||
head_dim=self.head_dim,
|
||||
nope_head_dim=self.nope_head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
out=output,
|
||||
)
|
||||
|
||||
def _forward_prefill(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
compressed_k_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> None:
|
||||
swa_only = attn_metadata is None
|
||||
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
# Use pre-computed prefill metadata.
|
||||
seq_lens = swa_metadata.prefill_seq_lens
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
assert seq_lens is not None
|
||||
assert gather_lens is not None
|
||||
|
||||
# Derive prefill-local token offsets from the full query_start_loc_cpu.
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
assert query_start_loc_cpu is not None
|
||||
assert query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
|
||||
if not swa_only:
|
||||
if self.compress_ratio == 4:
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
|
||||
topk_indices = topk_indices[:num_prefill_tokens]
|
||||
else:
|
||||
# C128A: pre-computed during metadata build.
|
||||
assert attn_metadata is not None
|
||||
topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
top_k = topk_indices.shape[-1]
|
||||
N = (self.max_model_len + self.compress_ratio - 1) // self.compress_ratio
|
||||
else:
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
|
||||
top_k = 0
|
||||
N = 0
|
||||
|
||||
M = N + self.window_size + self.max_num_batched_tokens
|
||||
chunk_size_const = self.PREFILL_CHUNK_SIZE
|
||||
num_chunks = (num_prefills + chunk_size_const - 1) // chunk_size_const
|
||||
|
||||
workspace_manager = current_workspace_manager()
|
||||
kv = workspace_manager.get_simultaneous(
|
||||
((chunk_size_const, M, q.shape[-1]), torch.bfloat16),
|
||||
)[0]
|
||||
for chunk_idx in range(num_chunks):
|
||||
chunk_start = chunk_idx * chunk_size_const
|
||||
chunk_end = min(chunk_start + chunk_size_const, num_prefills)
|
||||
chunk_size = chunk_end - chunk_start
|
||||
if not swa_only:
|
||||
# Gather compressed KV
|
||||
assert attn_metadata is not None
|
||||
block_table = attn_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
compressed_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
|
||||
gather_lens=None,
|
||||
block_table=block_table[chunk_start:chunk_end],
|
||||
block_size=attn_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
# Gather SWA KV
|
||||
swa_block_table = swa_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
swa_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end],
|
||||
gather_lens=gather_lens[chunk_start:chunk_end],
|
||||
block_table=swa_block_table[chunk_start:chunk_end],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=N,
|
||||
)
|
||||
|
||||
# Combine the topk indices and SWA indices for gathered KV cache
|
||||
query_start = (
|
||||
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
||||
)
|
||||
query_end = (
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices[query_start:query_end],
|
||||
query_start_loc[
|
||||
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
||||
],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
|
||||
kv_ws = kv[:chunk_size].reshape(-1, 1, q.shape[-1])
|
||||
out, _, _ = triton_bf16_mla_sparse_interface(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv_ws,
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
d_v=q.shape[-1],
|
||||
block_dpe=0,
|
||||
)
|
||||
output[query_start:query_end] = out
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""XPU sparse decode for DeepSeek V4 with FP8 KV cache.
|
||||
|
||||
Strategy: dequantize FP8 UE8M0 KV cache pages to BF16 on the fly,
|
||||
then reuse the BF16 sparse MLA attention kernel (xpu_sparse_mla_bf16).
|
||||
This keeps the external KV cache layout identical to CUDA/ROCm.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.xpu_mla_sparse import (
|
||||
triton_bf16_mla_sparse_interface,
|
||||
)
|
||||
|
||||
# FP8 DS MLA cache layout constants
|
||||
TOKEN_FP8_DIM = 448 # NoPE portion in FP8
|
||||
TOKEN_BF16_DIM = 64 # RoPE portion in BF16
|
||||
TOKEN_SCALE_DIM = 8 # UE8M0 scales per token
|
||||
QUANT_BLOCK_SIZE = 64 # Elements per quant block
|
||||
OUTPUT_DIM = 512 # = TOKEN_FP8_DIM + TOKEN_BF16_DIM after dequant
|
||||
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2 # 576 bytes per token
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequant_gather_slots_kernel(
|
||||
# Output workspace: [total_slots, OUTPUT_DIM] bf16
|
||||
out_ptr,
|
||||
# FP8 paged cache base pointer (uint8 flat)
|
||||
cache_ptr,
|
||||
# Global slot indices: [total_slots] int32
|
||||
indices_ptr,
|
||||
# Cache geometry
|
||||
cache_block_size: tl.constexpr,
|
||||
token_data_size: tl.constexpr, # 576
|
||||
block_stride: tl.int64, # k_cache.stride(0) — total uint8 per block
|
||||
fp8_dim: tl.constexpr, # 448
|
||||
bf16_dim: tl.constexpr, # 64
|
||||
scale_dim: tl.constexpr, # 8
|
||||
quant_block: tl.constexpr, # 64
|
||||
output_dim: tl.constexpr, # 512
|
||||
n_quant_blocks: tl.constexpr, # 7
|
||||
):
|
||||
"""Dequantize scattered FP8 slots into a flat BF16 workspace.
|
||||
|
||||
Grid: [total_slots] — one program per slot to gather.
|
||||
|
||||
Cache block layout (block_size tokens):
|
||||
[0, block_size*576): Token data, each token 448 FP8 + 128 BF16
|
||||
[block_size*576, block_size*576 + block_size*8): Scales
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
|
||||
# Load global slot index
|
||||
slot_idx = tl.load(indices_ptr + pid).to(tl.int64)
|
||||
|
||||
# Output pointer for this slot
|
||||
out_row_ptr = out_ptr + pid * output_dim
|
||||
|
||||
# Handle invalid slots (index < 0): write zeros
|
||||
if slot_idx < 0:
|
||||
zero = tl.zeros([quant_block], dtype=tl.bfloat16)
|
||||
for i in tl.static_range(0, 512, 64):
|
||||
offsets = i + tl.arange(0, quant_block)
|
||||
mask = offsets < output_dim
|
||||
tl.store(out_row_ptr + offsets, zero, mask=mask)
|
||||
return
|
||||
|
||||
# Compute block and position within block
|
||||
block_idx = slot_idx // cache_block_size
|
||||
pos_in_block = slot_idx % cache_block_size
|
||||
|
||||
# Block base pointer
|
||||
block_base = cache_ptr + block_idx * block_stride
|
||||
|
||||
# Token data: at offset pos_in_block * token_data_size within block
|
||||
token_data_ptr = block_base + pos_in_block * token_data_size
|
||||
|
||||
# Scale: after all token data, at offset
|
||||
# cache_block_size * token_data_size + pos_in_block * scale_dim
|
||||
scale_region_offset = tl.cast(cache_block_size, tl.int64) * token_data_size
|
||||
token_scale_ptr = block_base + scale_region_offset + pos_in_block * scale_dim
|
||||
|
||||
# ========== Dequantize FP8 portion (448 elements) ==========
|
||||
for qblock_idx in tl.static_range(n_quant_blocks):
|
||||
qblock_start = qblock_idx * quant_block
|
||||
offsets = qblock_start + tl.arange(0, quant_block)
|
||||
mask = offsets < fp8_dim
|
||||
|
||||
# Load FP8 as uint8 and bitcast
|
||||
x_uint8 = tl.load(token_data_ptr + offsets, mask=mask, other=0)
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
x_float = x_fp8.to(tl.float32)
|
||||
|
||||
# Load UE8M0 scale: scale = 2^(stored_value - 127)
|
||||
encoded_scale = tl.load(token_scale_ptr + qblock_idx)
|
||||
exponent = encoded_scale.to(tl.float32) - 127.0
|
||||
scale = tl.exp2(exponent)
|
||||
|
||||
# Dequantize and store as bf16
|
||||
x_dequant = x_float * scale
|
||||
tl.store(out_row_ptr + offsets, x_dequant.to(tl.bfloat16), mask=mask)
|
||||
|
||||
# ========== Copy BF16 portion (64 elements) directly ==========
|
||||
bf16_src_ptr = (token_data_ptr + fp8_dim).to(tl.pointer_type(tl.bfloat16))
|
||||
bf16_out_ptr = (out_row_ptr + fp8_dim).to(tl.pointer_type(tl.bfloat16))
|
||||
|
||||
for j in tl.static_range(bf16_dim // 16):
|
||||
chunk_offsets = j * 16 + tl.arange(0, 16)
|
||||
bf16_vals = tl.load(bf16_src_ptr + chunk_offsets)
|
||||
tl.store(bf16_out_ptr + chunk_offsets, bf16_vals)
|
||||
|
||||
|
||||
def dequant_gather_slots(
|
||||
out: torch.Tensor, # [total_slots, 512] bf16, pre-allocated
|
||||
cache: torch.Tensor, # [num_blocks, block_size, head_bytes] uint8
|
||||
indices: torch.Tensor, # [total_slots] int32, global slot IDs
|
||||
cache_block_size: int, # block_size for this cache
|
||||
) -> None:
|
||||
"""Dequantize FP8 UE8M0 pages at scattered slot indices into bf16."""
|
||||
total_slots = indices.shape[0]
|
||||
if total_slots == 0:
|
||||
return
|
||||
|
||||
block_stride = cache.stride(0)
|
||||
|
||||
_dequant_gather_slots_kernel[(total_slots,)](
|
||||
out,
|
||||
cache,
|
||||
indices,
|
||||
cache_block_size=cache_block_size,
|
||||
token_data_size=TOKEN_DATA_SIZE,
|
||||
block_stride=block_stride,
|
||||
fp8_dim=TOKEN_FP8_DIM,
|
||||
bf16_dim=TOKEN_BF16_DIM,
|
||||
scale_dim=TOKEN_SCALE_DIM,
|
||||
quant_block=QUANT_BLOCK_SIZE,
|
||||
output_dim=OUTPUT_DIM,
|
||||
n_quant_blocks=7,
|
||||
)
|
||||
|
||||
|
||||
def xpu_sparse_decode_fp8(
|
||||
q: torch.Tensor, # [num_tokens, num_heads, head_dim]
|
||||
kv_cache: torch.Tensor | None, # [num_blocks, block_size, head_bytes] uint8
|
||||
swa_kv_cache: torch.Tensor, # [num_blocks, swa_block_size, head_bytes] uint8
|
||||
swa_only: bool,
|
||||
topk_indices: torch.Tensor | None, # [num_tokens, 1, topk] global slot IDs
|
||||
topk_lens: torch.Tensor | None,
|
||||
swa_indices: torch.Tensor, # [num_tokens, 1, swa_k] global slot IDs
|
||||
swa_lens: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
out: torch.Tensor, # [num_tokens, num_heads, head_dim]
|
||||
) -> None:
|
||||
"""XPU decode: dequant FP8 pages to BF16, then BF16 sparse MLA attention.
|
||||
|
||||
Keeps external FP8 KV cache layout identical to CUDA/ROCm.
|
||||
Performance is slower due to on-the-fly dequant, but correctness is
|
||||
guaranteed by reusing the validated BF16 attention kernel.
|
||||
"""
|
||||
num_tokens = q.shape[0]
|
||||
device = q.device
|
||||
|
||||
if not swa_only and topk_indices is not None and kv_cache is not None:
|
||||
topk_idx_2d = (
|
||||
topk_indices.squeeze(1) if topk_indices.dim() == 3 else topk_indices
|
||||
)
|
||||
max_topk = topk_idx_2d.shape[1]
|
||||
else:
|
||||
topk_idx_2d = None
|
||||
max_topk = 0
|
||||
|
||||
use_topk = topk_idx_2d is not None
|
||||
swa_idx_2d = swa_indices.squeeze(1) if swa_indices.dim() == 3 else swa_indices
|
||||
max_swa = swa_idx_2d.shape[1]
|
||||
K_total = max_topk + max_swa
|
||||
|
||||
# Topk rows precede SWA rows in the flat workspace.
|
||||
n_topk_rows = num_tokens * max_topk
|
||||
workspace = torch.empty(
|
||||
(num_tokens * K_total, OUTPUT_DIM), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
if topk_idx_2d is not None and kv_cache is not None and n_topk_rows > 0:
|
||||
dequant_gather_slots(
|
||||
workspace[:n_topk_rows],
|
||||
kv_cache,
|
||||
topk_idx_2d.reshape(-1).to(torch.int32),
|
||||
kv_cache.shape[1],
|
||||
)
|
||||
|
||||
dequant_gather_slots(
|
||||
workspace[n_topk_rows:],
|
||||
swa_kv_cache,
|
||||
swa_idx_2d.reshape(-1).to(torch.int32),
|
||||
swa_kv_cache.shape[1],
|
||||
)
|
||||
|
||||
# Use a shape-derived width so replay accepts changing lens without host sync.
|
||||
_BLOCK_N = 16
|
||||
width = max(_BLOCK_N, ((K_total + _BLOCK_N - 1) // _BLOCK_N) * _BLOCK_N)
|
||||
|
||||
pos = torch.arange(width, device=device, dtype=torch.int32).unsqueeze(0)
|
||||
tok = torch.arange(num_tokens, device=device, dtype=torch.int32).unsqueeze(1)
|
||||
|
||||
slen = swa_lens.reshape(-1, 1).to(torch.int32).clamp(min=0, max=max_swa)
|
||||
if use_topk and topk_lens is not None:
|
||||
tlen = topk_lens.reshape(-1, 1).to(torch.int32).clamp(min=0, max=max_topk)
|
||||
else:
|
||||
tlen = torch.zeros_like(slen)
|
||||
|
||||
in_topk = pos < tlen
|
||||
in_swa = (pos >= tlen) & (pos < tlen + slen)
|
||||
|
||||
topk_ws = tok * max_topk + pos
|
||||
swa_ws = n_topk_rows + tok * max_swa + (pos - tlen)
|
||||
|
||||
combined_indices = torch.where(
|
||||
in_topk,
|
||||
topk_ws,
|
||||
torch.where(in_swa, swa_ws, torch.full_like(topk_ws, -1)),
|
||||
)
|
||||
|
||||
out_attn, _, _ = triton_bf16_mla_sparse_interface(
|
||||
q=q,
|
||||
kv=workspace.unsqueeze(1),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=softmax_scale,
|
||||
d_v=q.shape[-1],
|
||||
block_dpe=0,
|
||||
)
|
||||
out.copy_(out_attn)
|
||||
@@ -1,20 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1 hardware-isolated model entry point."""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quant_config import DeepseekV4FP8Config
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from .amd.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from .amd.vl_model import DeepseekV41ForCausalLM
|
||||
else:
|
||||
from .nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from .nvidia.vl_model import DeepseekV41ForCausalLM
|
||||
|
||||
__all__ = [
|
||||
"DSparkDeepseekV4ForCausalLM",
|
||||
"DeepseekV4FP8Config",
|
||||
"DeepseekV41ForCausalLM",
|
||||
]
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,533 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark draft model for DeepSeek-V4.1 (semi-autoregressive speculative decoding).
|
||||
|
||||
See: qwen3_dspark.py for base architecture. This one is specialized to the
|
||||
DSV4.1 DSpark, which reuses the target model's architecture similarly to MTP.
|
||||
Its weights ship in the target checkpoint under the ``mtp.{0,1,2}.*`` prefix.
|
||||
|
||||
To implement non-causal attention, we leverage the sparse attention implementation to
|
||||
include the future query tokens in the top-k indices for each query token.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
_linear_scale_param_name,
|
||||
_use_sequence_parallel,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
|
||||
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DSparkDeepseekV4Model(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_eps = config.hc_eps
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.num_dspark_layers = (
|
||||
getattr(config, "n_mtp_layers", None)
|
||||
or getattr(config, "num_nextn_predict_layers", None)
|
||||
or 3
|
||||
)
|
||||
|
||||
# Shared with the target (aliased by the speculator's loading utility).
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
self.main_proj = ReplicatedLinear(
|
||||
config.hidden_size * len(self.target_layer_ids),
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "main_proj"),
|
||||
)
|
||||
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4DecoderLayer(
|
||||
current_vllm_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
)
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
)
|
||||
|
||||
# Heads: final norm, and the Markov + confidence heads.
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target
|
||||
# checkpoint. v4.1 has no learned hc_head: the hc copies are
|
||||
# collapsed with a pre-mix derived from the last layer's hc_ffn
|
||||
# projection at the end of forward().
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", True):
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
config.hidden_size + config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""main_x = main_norm(main_proj(concat of target aux hidden states)).
|
||||
|
||||
``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)].
|
||||
"""
|
||||
return self.main_norm(self.main_proj(aux_hidden_states))
|
||||
|
||||
@torch.inference_mode()
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
main_x: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Insert the sliding-window context KV for every draft layer.
|
||||
|
||||
Mirrors the reference DSparkAttention: each layer derives its context KV
|
||||
from the SAME projected target hidden ``main_x``, via that layer's own
|
||||
``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the
|
||||
layer's context slots.
|
||||
|
||||
``context_slot_mappings`` is a per-layer list (each entry is the context
|
||||
slot mapping for that layer's kv-cache group, since the hybrid manager may
|
||||
place draft layers in different groups). ``None`` (or a ``None`` entry)
|
||||
runs the projection to reserve workspace but writes nothing (profiling).
|
||||
"""
|
||||
for i, layer in enumerate(self.layers):
|
||||
slot_mapping = (
|
||||
None if context_slot_mappings is None else context_slot_mappings[i]
|
||||
)
|
||||
attn = layer.attn
|
||||
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
|
||||
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
|
||||
qr_kv, _ = attn.fused_wqa_wkv(main_x)
|
||||
kv = qr_kv[..., attn.q_lora_rank :]
|
||||
kv = attn.kv_norm(kv)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
_insert_context_kv(attn, kv, context_positions, slot_mapping)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, inputs_embeds
|
||||
)
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
input_ids = sp_shard(input_ids)
|
||||
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
|
||||
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
|
||||
residual = post_mix = res_mix = pre_mix = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
hidden_states = mhc_post_torch(hidden_states, residual, post_mix, res_mix)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
pre_mix = sp_all_gather(pre_mix)[:full_num_tokens]
|
||||
# Collapse the hc copies with the pre-mix from the last layer's FFN
|
||||
# mixes — the mix the reference forward_head applies (hc_pre with the
|
||||
# last block's ffn pre-mix). Return the PRE-norm head hidden;
|
||||
# compute_logits applies self.norm.
|
||||
hidden_states = DeepseekV4DecoderLayer._hc_collapse(hidden_states, pre_mix)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _insert_context_kv(
|
||||
attn: nn.Module,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""RoPE + quant + paged-cache insert of (already kv_norm'd) context KV.
|
||||
|
||||
Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy
|
||||
query and discard it, since context tokens have no query). Mirrors
|
||||
``DeepseekV4Attention._fused_qnorm_rope_kv_insert``.
|
||||
"""
|
||||
swa_cache = attn.swa_cache_layer.kv_cache
|
||||
block_size = attn.swa_cache_layer.block_size
|
||||
cos_sin_cache = attn.rotary_emb.cos_sin_cache
|
||||
cache_dtype = swa_cache.dtype
|
||||
n_ctx = kv.shape[0]
|
||||
dummy_q = torch.zeros(
|
||||
(n_ctx, attn.n_local_heads, attn.head_dim),
|
||||
dtype=kv.dtype,
|
||||
device=kv.device,
|
||||
)
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged layout
|
||||
swa_2d = swa_cache.view(swa_cache.shape[0], -1)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.padded_heads,
|
||||
attn.eps,
|
||||
block_size,
|
||||
True, # apply_q_norm (unused: the query is a discarded dummy)
|
||||
attn.kv_mxfp8,
|
||||
)
|
||||
elif cache_dtype == torch.bfloat16:
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
else: # per-tensor fp8 (torch.float8_e4m3fn)
|
||||
# TODO(ben): double-check if this is being dispatched correctly for FI backend
|
||||
swa_3d = swa_cache.view(-1, block_size, attn.head_dim)
|
||||
dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
dummy_q,
|
||||
kv,
|
||||
dummy_q_fp8,
|
||||
swa_3d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
attn._flashinfer_fp8_kv_scale,
|
||||
attn._flashinfer_fp8_q_scale_inv,
|
||||
attn.eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
has_own_lm_head = False
|
||||
# Full-vocab draft: draft ids are target ids, no remapping needed.
|
||||
draft_id_to_target_id = None
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.linear_scale_name = _linear_scale_param_name(
|
||||
vllm_config, getattr(self.config, "expert_dtype", "fp4")
|
||||
)
|
||||
self.pad_shared_expert = getattr(
|
||||
self.quant_config, "weight_block_size", None
|
||||
) is not None and not _use_sequence_parallel(vllm_config)
|
||||
self.model = DSparkDeepseekV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
# Shared with the target (aliased by the speculator's load utility).
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.combine_hidden_states(aux_hidden_states)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
# DSV4 MLA path: each draft layer's sliding-window cache is a separate
|
||||
# layer, named by its prefix.
|
||||
return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers]
|
||||
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mappings: list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
self.model.precompute_and_store_context_kv(
|
||||
context_states, context_positions, context_slot_mappings
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Returns the pre-norm collapsed head hidden ([T, hidden_size]).
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Base logits U_k = lm_head(norm(head_hidden))."""
|
||||
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Full-vocab draft: base logits, no d2t scatter.
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids # full-vocab: draft ids are target ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
# --- Weight loading ----------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint.
|
||||
|
||||
Non-mtp weights (embed/head/main layers) belong to the target model and
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
# Draft MoE layers use the dspark_* expert counts, not the
|
||||
# backbone's (see DeepseekV4MoE and the reference
|
||||
# ModelArgs.get_moe_config).
|
||||
n_draft_experts = (
|
||||
getattr(self.config, "dspark_n_routed_experts", 0)
|
||||
or self.config.n_routed_experts
|
||||
)
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=n_draft_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
if getattr(self.config, "expert_dtype", "fp4") == "fp4"
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
|
||||
# (param_name, ckpt_shard_name, shard_id) for non-expert stacked params.
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
if "confidence_head." in name:
|
||||
loaded_confidence_head = True
|
||||
|
||||
# ``.scale`` -> per-method scale suffix.
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
expert_scale_suffix
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else f".{self.linear_scale_name}"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
|
||||
if self.pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = DeepseekV4Model._pad_shared_expert_weight(
|
||||
self.quant_config, name, loaded_weight
|
||||
)
|
||||
|
||||
# E8M0 expert scales: keep raw exponent bytes.
|
||||
if ".experts." in name:
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
success = param.weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
|
||||
# Stacked rules only apply to decoder-layer weights. Head-stack params
|
||||
# (main_proj/norm/markov_head/confidence_head) load directly —
|
||||
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
self.process_weights_after_loading()
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# ROCm linears and fused-MoE runners finalize their own quantized
|
||||
# parameters. NVIDIA-only MegaMoE/WO-A requantization is not needed.
|
||||
return
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
Returns None for non-mtp weights (owned by the target model).
|
||||
"""
|
||||
m = re.match(r"mtp\.(\d+)\.(.*)", name)
|
||||
if m is None:
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
|
||||
return None
|
||||
# The checkpoint calls the Markov head's factors ``embed``/``head``;
|
||||
# DSparkMarkovHead registers them as ``markov_w1``/``markov_w2``.
|
||||
if rest.startswith("markov_head.embed."):
|
||||
return "model.markov_head.markov_w1." + rest.removeprefix(
|
||||
"markov_head.embed."
|
||||
)
|
||||
if rest.startswith("markov_head.head."):
|
||||
return "model.markov_head.markov_w2." + rest.removeprefix(
|
||||
"markov_head.head."
|
||||
)
|
||||
# Head-stack params live at model level (mtp.last), context combiner at
|
||||
# model level (mtp.0); everything else is a per-layer decoder block.
|
||||
head_prefixes = (
|
||||
"norm.",
|
||||
"markov_head.",
|
||||
"confidence_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
):
|
||||
return f"model.{rest}"
|
||||
return f"model.layers.{stage}.{rest}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,97 +0,0 @@
|
||||
# DeepSeek-V4.1-Flash mono decode layer (gfx950, FlyDSL)
|
||||
|
||||
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
|
||||
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
|
||||
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
|
||||
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
|
||||
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
|
||||
ATOM's V4.1 mono decode
|
||||
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
|
||||
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
|
||||
from it, each file naming its origin; the attention stages (`attention/`) are
|
||||
new, reading vLLM's KV records and weights as loaded.
|
||||
|
||||
## Scope
|
||||
|
||||
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
|
||||
settled, without folding the previous output in, on layer 0 (the embedding
|
||||
broadcast to the four streams) and after an Engram write (layers 1 and 14,
|
||||
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
|
||||
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
|
||||
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
|
||||
key table from that top-k. The FFN launch serves a layer outside these (an
|
||||
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
|
||||
The paths interleave freely (same tensors at the layer boundary).
|
||||
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
|
||||
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
|
||||
routing bias is `e_score_correction_bias`).
|
||||
- Contract: a drop-in for `DeepseekV4DecoderLayer.forward`:
|
||||
in `x` (the previous FFN output, reduced, bf16 [M, 5120]), `residual`
|
||||
(bf16 [M, 4, 5120]), `post_mix` [M, 4, 1], `res_mix` [M, 4, 4],
|
||||
`pre_mix` [M, 4] (f32, the previous seam's)
|
||||
out the MoE output (reduced), the residual after the FFN seam, and that
|
||||
seam's post / comb / pre mixes.
|
||||
|
||||
## Launches
|
||||
|
||||
K1 seam (stages/seam.py): slice (160 x 32 columns): fold x into the
|
||||
residual (post x + sum comb r), the collapsed input (pre . R'), partial
|
||||
mixes (fn . R') and sums of squares -> gate (S): 24 mixes -> pre / post
|
||||
/ comb (Sinkhorn 20) -> norm (S): attn_norm -> vLLM MXFP8 -> X8
|
||||
attention front (attention/front.py): wqkv, q / kv norms + KV insert,
|
||||
wq_b + RoPE -> Q
|
||||
K2 attention back (attention/back.py): split attention, combine + inverse
|
||||
RoPE, wo_a, wo_b; the bf16 wo_b partial pushed to every rank
|
||||
FFN seam: reduce (the ranks' partials summed in rank order, fp32 ->
|
||||
bf16: the attention output), slice, gate, norm -> the MoE input
|
||||
MoE (stages/moe.py): xq, router (bf16 GEMV, f32 logits), route
|
||||
(sqrtsoftplus + bias, top-6, renormalized x 1.5), shared expert (MXFP8
|
||||
GEMVs, clamped SiLU), ug (MXFP4 x MXFP8 per expert of the step's
|
||||
union), down (+ routing weight, top-k order; + shared) -> push, rank-order
|
||||
sum -> out
|
||||
FFN K2 without the attention back: push (160 x 32 columns) of vLLM's
|
||||
unreduced wo_b output, as its bits, to every rank's attention region,
|
||||
then K2's FFN seam and MoE (the same code, `layer.run_ffn`)
|
||||
|
||||
## Execution model and hand-offs
|
||||
|
||||
- Grid = 256 CTAs (one a CU, all resident), 512 threads a CTA, LDS a per-stage
|
||||
union. A stage is a task table: CTA `b` runs tasks `b, b + 256, ...`, stages
|
||||
in order; a consumer only waits on earlier stages, so no deadlock while the
|
||||
grid is resident. A GEMV task issues its weight loads before it polls its
|
||||
activations.
|
||||
- In a launch: tagged pairs (value + tag in one 8 B store) or plain data +
|
||||
flags, at device scope. The tag is the launch pair's epoch (K1 `2e + 1`,
|
||||
K2 or the FFN launch `2e + 2`), a device word their CTA 0 moves on once every
|
||||
CTA has marked the launch done: no clearing between steps, no host step hook, every argument
|
||||
a fixed device pointer (graph-safe).
|
||||
- Across ranks: one symmetric uncached buffer a rank (aiter `UncachedIpcHeap`),
|
||||
every rank holding every peer's address; pushes are system-scope stores of
|
||||
(value, tag) pairs. Every rank runs the same launches, so the epochs agree;
|
||||
peer regions alternate by epoch parity, and a rank cannot run two steps ahead
|
||||
of a peer (each step's reduce waits on every rank's push).
|
||||
|
||||
## Numerics (the vLLM path each stage reproduces)
|
||||
|
||||
- Seams: aiter `mhc_fused_post_pre_delayed_rmsnorm` (post fold order, fn as
|
||||
bf16 hi + lo, products summed in order, Triton sigmoid / exp / rcp).
|
||||
- Attention: vLLM's ROCm chain. `mxfp8(x)`: one E8M0 per 32 values,
|
||||
`code = clamp(ceil(log2(amax / 448)) + 127, 0, 254)`; weights in vLLM's
|
||||
row-major e4m3 [N, K] with [N/32, K/32] E8M0 scales (one shared copy); KV
|
||||
records as vLLM's insert writes them (dims 0..447 e4m3 with a UE8M0 per 64,
|
||||
RoPE dims bf16, 584 B); keys = the sliding window plus the top-k compressed
|
||||
rows, softmax in base 2 with the attention sink folded in.
|
||||
- MoE: vLLM's router (bf16 gate GEMV, f32 logits, `topk_hash_softplus_sqrt`)
|
||||
and AITER a8w4 `fused_moe` (interleaved gate / up, swiglu limit 10, weights
|
||||
in aiter's (16, 16) shuffle and `shuffle_scale` order); the shared expert as
|
||||
vLLM's MXFP8 block-32 linears with `silu_and_mul_with_clamp`.
|
||||
- All-reduces: exact, fp32 in rank order -> bf16. vLLM's decode all-reduces at
|
||||
M <= 48 are below its INT4 quick-reduce threshold, so vLLM runs them exact too.
|
||||
|
||||
## Validation
|
||||
|
||||
- Per layer against vLLM's own modules in a TP process group (`DeepseekV4MoE`,
|
||||
the mHC seams, the attention chain, the all-reduce), real checkpoint weights.
|
||||
- Bench: layers of one type back to back in one HIP graph, cold weights,
|
||||
us per layer, M = 1..48, TP2 / TP4; then e2e (AgentX, InferenceX#3690
|
||||
recipe).
|
||||
@@ -1,5 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1-Flash mono decode layer: one decoder layer of a decode step in
|
||||
two persistent FlyDSL launches, its TP all-reduces in-kernel (README.md). The
|
||||
host side is ``runner``; vLLM docks it in ``..mono_decode``."""
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The mono layer's attention stages: ``front`` (K1: wqkv, q / kv norms and the
|
||||
KV insert, wq_b) and ``back`` (K2: split attention, combine, wo_a, wo_b)."""
|
||||
@@ -1,888 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K2 ``back``: one DeepSeek-V4.1 attention layer from the rotated query to the
|
||||
rank's wo_b partial.
|
||||
|
||||
One launch a layer and rank, ``BLOCKS`` x ``THREADS``, S rows:
|
||||
|
||||
split (S x chunks of CK keys): flash-decoding over one chunk of a token's
|
||||
keys (K1's key table: top-k compressed rows, then window rows) for
|
||||
every head of the rank. The token's q in LDS; each wave one
|
||||
(head tile, 16-key tile): its fp8_ds_mla rows loaded once,
|
||||
dequantized into its QK operands and kept raw in LDS; QK (bf16
|
||||
MFMA), the chunk's max / sum per head, bf16 p in LDS; PV over the
|
||||
chunk's raw rows (wave w: dims 64 w ..) -> the chunk's partial
|
||||
(bf16) and its (max, sum) per head, published
|
||||
combine (S x H): a (token, head)'s chunks merged with the sink folded in,
|
||||
bf16, inverse GPT-J RoPE of dims 448.., vLLM's MXFP8 of each 32
|
||||
-> XO (published)
|
||||
wo_a (G x 1024 / 32, two 16-row tiles): the group's XO rows times wo_a
|
||||
-> bf16 -> MXFP8 of the 32 rows -> X8B (published)
|
||||
wo_b (320, 16 rows): X8B times wo_b -> bf16 -> ``out``
|
||||
|
||||
Hand-offs are plain device-scope data behind tagged flags (half the bytes of a
|
||||
tagged pair per value). Adapted from ATOM's V4.1 ``attn_post`` (K2a), which
|
||||
reads a bf16 pool and ends in a peer all-reduce.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
from .device import (
|
||||
BLOCKS,
|
||||
CM_DEV,
|
||||
THREADS,
|
||||
WAVES,
|
||||
Mailbox,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
clamp_fp8,
|
||||
first_task,
|
||||
fp8_pack4,
|
||||
fp8x8_bf16,
|
||||
fresh,
|
||||
gload,
|
||||
gstore,
|
||||
hw_exp2,
|
||||
lane_gather,
|
||||
ld_f32,
|
||||
ld_i32,
|
||||
mfma_bf16,
|
||||
mx_code,
|
||||
mx_mul,
|
||||
pow2,
|
||||
rsrc,
|
||||
spin_until,
|
||||
stamp,
|
||||
traced,
|
||||
xshfl,
|
||||
)
|
||||
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
|
||||
from .plan import (
|
||||
DATA,
|
||||
HALF,
|
||||
HEAD_DIM,
|
||||
HEAD_TILE,
|
||||
KEYS,
|
||||
NOPE,
|
||||
ROPE,
|
||||
ROWS,
|
||||
WOA_ROWS,
|
||||
Dims,
|
||||
back_scratch,
|
||||
cdiv,
|
||||
front_end,
|
||||
tile_rows,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
_STREAM = fx.Stream(None)
|
||||
NEG = -3.4028234663852886e38
|
||||
LOG2E = 1.4426950408889634
|
||||
REC = DATA + 8 # a raw record row in LDS: 576 data bytes + 8 scale bytes
|
||||
Q_ROW = (
|
||||
HEAD_DIM // 2 + 4
|
||||
) # a q row in LDS (i32 words): 512 bf16 + 16 B, no bank aliasing
|
||||
P_PAD = 2 # i32 words past a P row: 16 head rows read together hit distinct banks
|
||||
# wo_a's K order: waves 0..3 / 4..7 take the two 16-row tiles, a contiguous K
|
||||
# range a wave, summed in wave order
|
||||
WOA_SPLIT_K = WAVES * ROWS // WOA_ROWS
|
||||
BACK_POINTS = 16 # start, stage ends, split internals, GEMV internals
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackBuild:
|
||||
tokens: int
|
||||
tp: int
|
||||
ratio: int # 0: window only; 1 / 2: top-k over a compressed cache
|
||||
timeline: bool = False
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
def head_groups(d: Dims, s: int) -> int:
|
||||
"""Wave groups a split unit's head tiles are spread over: two (a tile each,
|
||||
64-key chunks, twice the units) while the units fit the grid, else one (a
|
||||
wave's keys loaded and dequantized once for every head tile)."""
|
||||
return 2 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= BLOCKS else 1
|
||||
|
||||
|
||||
def chunk_keys(d: Dims, s: int) -> int:
|
||||
"""A split unit's keys: a 16-key tile a wave of a head group."""
|
||||
return 16 * (WAVES // head_groups(d, s))
|
||||
|
||||
|
||||
def p_row(d: Dims, s: int) -> int:
|
||||
"""A P row (one head's chunk of bf16 p) in LDS, i32 words."""
|
||||
return chunk_keys(d, s) // 2 + P_PAD
|
||||
|
||||
|
||||
@traced
|
||||
def publish(c, region, i):
|
||||
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if c["tid"] == 0:
|
||||
c["mb"].put(region, i, fx.Int32(1))
|
||||
|
||||
|
||||
@traced
|
||||
def await_flags(c, region, n, index):
|
||||
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
|
||||
tid = c["tid"]
|
||||
if tid < n:
|
||||
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- keys
|
||||
|
||||
|
||||
def key_rec(c, t, k):
|
||||
"""(data address, scale address, valid) of token t's key k from K1's key
|
||||
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
|
||||
none (it reads a zero record: its scores are masked, its values zeros)."""
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
# a table this launch wrote (``krdy``) is read past the CU's cache
|
||||
cm = CM_DEV if const_expr("krdy" in c) else 0
|
||||
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
|
||||
ok = val != fx.Int32(-1)
|
||||
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
|
||||
if const_expr(ratio > 0):
|
||||
is_c = ok & (val < 0)
|
||||
blk = is_c.select(a["comp_block"], a["swa_block"])
|
||||
stride = is_c.select(a["comp_stride"], a["swa_stride"])
|
||||
base = is_c.select(a["comp"], a["swa"])
|
||||
else:
|
||||
blk, stride, base = a["swa_block"], a["swa_stride"], a["swa"]
|
||||
b = slot // blk
|
||||
off = slot % blk
|
||||
page = base + fx.Int64(b) * stride
|
||||
z = a["zrec"]
|
||||
data = ok.select(page + fx.Int64(off * DATA), z)
|
||||
sc = ok.select(page + fx.Int64(blk * DATA + off * 8), z + fx.Int64(DATA))
|
||||
return data, sc, ok
|
||||
|
||||
|
||||
def _scale_byte(scw, blk64):
|
||||
return (scw[blk64 // 4] >> (blk64 % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def _fp8_pair_f32(word, scale, hi):
|
||||
v = rocdl.cvt_scalef32_pk_f32_fp8(
|
||||
fx.Vector.make_type(2, fx.Float32),
|
||||
as_ir_value(fx.Int32(word)),
|
||||
as_ir_value(fx.Float32(scale)),
|
||||
hi,
|
||||
)
|
||||
v = fx.Vector(v)
|
||||
return [v[0], v[1]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- split
|
||||
|
||||
|
||||
@traced
|
||||
def stage_split(c, unit):
|
||||
"""Flash-decoding over chunk ``unit % nchunk`` of token ``unit // nchunk``,
|
||||
every head of the rank -> the chunk's partial and (max, sum) per head. Wave
|
||||
(head group hg, key tile kt): keys 16 kt .. of the chunk (loaded and
|
||||
dequantized once) against the group's head tiles; then dims 64 w .. of the
|
||||
PV over every key and head."""
|
||||
tid, lane, wave, d, s = c["tid"], c["lane"], c["wave"], c["d"], c["S"]
|
||||
a, sl = c["args"], c["sl"]
|
||||
nchunk = c["nchunk"]
|
||||
t = unit // nchunk
|
||||
ch = unit % nchunk
|
||||
ck = chunk_keys(d, s)
|
||||
ngrp = head_groups(d, s)
|
||||
nkt = WAVES // ngrp # 16-key tiles a chunk
|
||||
nht = d.head_tiles // ngrp # head tiles a wave
|
||||
hgi = wave // nkt
|
||||
kt = wave % nkt
|
||||
g = lane // 16
|
||||
r16 = lane % 16
|
||||
k0 = ch * ck
|
||||
if const_expr("krdy" in c):
|
||||
await_flags(c, c["krdy"], 1, lambda u: t)
|
||||
# the key count, this lane's key and q: independent loads, out together
|
||||
# (the key table holds -1 past a token's keys)
|
||||
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
|
||||
key = k0 + kt * 16 + r16
|
||||
data, sc, okv = key_rec(c, t, key)
|
||||
qrow = d.heads * HEAD_DIM // 8
|
||||
qv = []
|
||||
for i in range_constexpr(cdiv(qrow, THREADS)):
|
||||
u = fresh(tid) + THREADS * i
|
||||
qv.append(
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(a["q"]),
|
||||
(t * d.q_rows) // 2 + fx.min(u, qrow - 1) * 4,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
)
|
||||
)
|
||||
if k0 < kv_len:
|
||||
# a key's 448 e4m3 as 7 x 16 B (lane g: dims 64 c + 16 g ..), its 64
|
||||
# bf16 rope dims as 2 x 16 B (32 c + 8 g ..), its 7 scale bytes
|
||||
scw = gload(sc, words=2)
|
||||
nope = [
|
||||
gload(data + fx.Int64(64 * c4 + 16 * g), words=4)
|
||||
for c4 in range(NOPE // 64)
|
||||
]
|
||||
rope = [
|
||||
gload(data + fx.Int64(NOPE + (cc * 32 + 8 * g) * 2), words=4)
|
||||
for cc in range(ROPE // 32)
|
||||
]
|
||||
# q of every head into LDS (bf16 rows of 512 + pad), 16 B a thread
|
||||
for i in range_constexpr(cdiv(qrow, THREADS)):
|
||||
u = fresh(tid) + THREADS * i
|
||||
if u < qrow:
|
||||
fx.ptr_store(
|
||||
qv[i],
|
||||
sl["q"]
|
||||
+ (u // (HEAD_DIM // 8)) * Q_ROW
|
||||
+ (u % (HEAD_DIM // 8)) * 4,
|
||||
)
|
||||
# the raw rows to LDS for the PV (head group 0's waves)
|
||||
if hgi == 0:
|
||||
rawb = sl["raw"] + (kt * 16 + r16) * (REC // 4)
|
||||
for c4 in range_constexpr(NOPE // 64):
|
||||
fx.ptr_store(nope[c4], rawb + (64 * c4 + 16 * g) // 4)
|
||||
for cc in range_constexpr(ROPE // 32):
|
||||
fx.ptr_store(rope[cc], rawb + (NOPE + (cc * 32 + 8 * g) * 2) // 4)
|
||||
if g == 0:
|
||||
fx.ptr_store(scw, rawb + DATA // 4)
|
||||
gpu.barrier()
|
||||
stamp(c, 5)
|
||||
# QK: A = the keys (dequantized once), B = q of a head tile (LDS); the
|
||||
# contraction's 32-dim chunk 2 c + h of the nope dims is {64 c + 16 g +
|
||||
# 8 h ..} (lane g's 16 B load), q read in the same order
|
||||
scale = fx.Int32(a["qk_scale"]).bitcast(fx.Float32)
|
||||
st = sl["st"]
|
||||
accs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(nht)]
|
||||
for c4 in range_constexpr(NOPE // 64):
|
||||
ksc = pow2(_scale_byte(scw, c4))
|
||||
for h2 in range_constexpr(2):
|
||||
kw = fp8x8_bf16(nope[c4][2 * h2], nope[c4][2 * h2 + 1], ksc).bitcast(
|
||||
fx.BFloat16
|
||||
)
|
||||
for ht in range_constexpr(nht):
|
||||
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
|
||||
qw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
qb + 32 * c4 + 8 * g + 4 * h2,
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
|
||||
for cc in range_constexpr(ROPE // 32):
|
||||
kw = rope[cc].bitcast(fx.BFloat16)
|
||||
for ht in range_constexpr(nht):
|
||||
qb = sl["q"] + ((hgi * nht + ht) * HEAD_TILE + r16) * Q_ROW
|
||||
qw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
qb + (NOPE + cc * 32 + 8 * g) // 2,
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
accs[ht] = mfma_bf16(kw, qw.bitcast(fx.BFloat16), accs[ht])
|
||||
# scores of keys 4 g + i (C rows), head r16 of a tile (C column)
|
||||
ok = (key < kv_len) & okv
|
||||
okw = ok.select(fx.Int32(1), fx.Int32(0))
|
||||
valid = [lane_gather(okw, 4 * g + i) != 0 for i in range(4)]
|
||||
svs = []
|
||||
for ht in range_constexpr(nht):
|
||||
sv = [
|
||||
fx.Float32(valid[i].select(accs[ht][i] * scale, fx.Float32(NEG)))
|
||||
for i in range(4)
|
||||
]
|
||||
mw = butterfly(
|
||||
fx.max(fx.max(sv[0], sv[1]), fx.max(sv[2], sv[3])), (32, 16), fx.max
|
||||
)
|
||||
if g == 0:
|
||||
fx.ptr_store(mw, st + ((hgi * nht + ht) * nkt + kt) * HEAD_TILE + r16)
|
||||
svs.append(sv)
|
||||
gpu.barrier()
|
||||
lst = st + d.heads * nkt
|
||||
for ht in range_constexpr(nht):
|
||||
hti = hgi * nht + ht
|
||||
m = fx.ptr_load(st + (hti * nkt) * HEAD_TILE + r16)
|
||||
for j in range_constexpr(1, nkt):
|
||||
m = fx.max(m, fx.ptr_load(st + (hti * nkt + j) * HEAD_TILE + r16))
|
||||
sv = svs[ht]
|
||||
p = [
|
||||
fx.Float32(valid[i].select(hw_exp2(sv[i] - m), fx.Float32(0.0)))
|
||||
for i in range(4)
|
||||
]
|
||||
lw = (p[0] + p[2]) + (p[1] + p[3])
|
||||
lw = lw + xshfl(lw, 32)
|
||||
lw = lw + xshfl(lw, 16)
|
||||
pw = (
|
||||
fx.Vector.from_elements(p, fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)
|
||||
)
|
||||
pbase = (
|
||||
sl["p"] + (hti * HEAD_TILE + r16) * p_row(d, s) + (kt * 16 + 4 * g) // 2
|
||||
)
|
||||
fx.ptr_store(fx.Vector.from_elements([pw[0], pw[1]], fx.Int32), pbase)
|
||||
if g == 0:
|
||||
fx.ptr_store(lw, lst + (hti * nkt + kt) * HEAD_TILE + r16)
|
||||
gpu.barrier()
|
||||
# chunk stats: (max, sum) per head
|
||||
if tid < d.heads:
|
||||
hts = tid // HEAD_TILE
|
||||
hr = tid % HEAD_TILE
|
||||
mm = fx.ptr_load(st + (hts * nkt) * HEAD_TILE + hr)
|
||||
ll = fx.ptr_load(lst + (hts * nkt) * HEAD_TILE + hr)
|
||||
for j in range_constexpr(1, nkt):
|
||||
mm = fx.max(mm, fx.ptr_load(st + (hts * nkt + j) * HEAD_TILE + hr))
|
||||
ll = ll + fx.ptr_load(lst + (hts * nkt + j) * HEAD_TILE + hr)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([mm, ll], fx.Float32),
|
||||
rsrc(c["pstat"]),
|
||||
((t * nchunk + ch) * d.heads + tid) * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
# PV: wave w dims 64 w + 4 r16 + dt (dt = 0..3: one LDS word holds the
|
||||
# four of a key); the rope wave (dims 448..) reads bf16 pairs. Lane
|
||||
# (r16, g) takes keys 4 g + j of a 16-key tile: A[dim][key]
|
||||
is_rope = wave == WAVES - 1
|
||||
dbase = 64 * wave
|
||||
sh = (wave % 4) * 8
|
||||
accs = [
|
||||
[fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(4)]
|
||||
for _ in range(d.head_tiles)
|
||||
]
|
||||
for kt2 in range_constexpr(nkt):
|
||||
rb0 = sl["raw"] + (kt2 * 16 + 4 * g) * (REC // 4)
|
||||
vv = fx.Vector.filled(16, 0.0, fx.Float32) # [dt][j]
|
||||
if is_rope:
|
||||
rv = []
|
||||
for j in range_constexpr(4):
|
||||
rw = fx.Vector(
|
||||
fx.ptr_load(
|
||||
rb0 + j * (REC // 4) + (NOPE + 8 * r16) // 4,
|
||||
result_type=fx.Vector.make_type(2, fx.Int32),
|
||||
)
|
||||
)
|
||||
rv.append([bf_lo(rw[0]), bf_hi(rw[0]), bf_lo(rw[1]), bf_hi(rw[1])])
|
||||
vv = fx.Vector.from_elements(
|
||||
[rv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
|
||||
)
|
||||
else:
|
||||
nv = []
|
||||
for j in range_constexpr(4):
|
||||
rb = rb0 + j * (REC // 4)
|
||||
w = fx.Int32(fx.ptr_load(rb + (dbase + 4 * r16) // 4))
|
||||
scb = fx.Int32(fx.ptr_load(rb + DATA // 4 + wave // 4))
|
||||
scl = pow2((scb >> sh) & 0xFF)
|
||||
nv.append(
|
||||
_fp8_pair_f32(w, scl, False) + _fp8_pair_f32(w, scl, True)
|
||||
)
|
||||
vv = fx.Vector.from_elements(
|
||||
[nv[j][dt] for dt in range(4) for j in range(4)], fx.Float32
|
||||
)
|
||||
for ht2 in range_constexpr(d.head_tiles):
|
||||
pb = fx.Vector(
|
||||
fx.ptr_load(
|
||||
sl["p"]
|
||||
+ (ht2 * HEAD_TILE + r16) * p_row(d, s)
|
||||
+ (kt2 * 16 + 4 * g) // 2,
|
||||
result_type=fx.Vector.make_type(2, fx.Int32),
|
||||
)
|
||||
).bitcast(fx.Int16)
|
||||
for dt in range_constexpr(4):
|
||||
av = (
|
||||
fx.Vector.from_elements(
|
||||
[vv[dt * 4 + j] for j in range(4)], fx.Float32
|
||||
)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int16)
|
||||
)
|
||||
accs[ht2][dt] = fx.Vector(
|
||||
rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.vec(4, T.f32), [av, pb, accs[ht2][dt]]
|
||||
)
|
||||
)
|
||||
stamp(c, 6)
|
||||
# partial[t][ch][head][dim] bf16: lane (r16, g) holds C rows 4 g + i ->
|
||||
# dims 64 w + 16 g + 4 i + dt of head ht2 * 16 + r16: 32 contiguous
|
||||
# bytes, two 16 B stores (4 lanes fill a 128 B line)
|
||||
for ht2 in range_constexpr(d.head_tiles):
|
||||
hh = ht2 * HEAD_TILE + r16
|
||||
words = []
|
||||
for i in range_constexpr(4):
|
||||
v4 = fx.Vector.from_elements(
|
||||
[accs[ht2][dt][i] for dt in range(4)], fx.Float32
|
||||
)
|
||||
w2 = v4.to(fx.BFloat16).bitcast(fx.Int32)
|
||||
words += [w2[0], w2[1]]
|
||||
off = (((t * nchunk + ch) * d.heads + hh) * HEAD_DIM + dbase + 16 * g) // 2
|
||||
for hlf in range_constexpr(2):
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words[4 * hlf : 4 * hlf + 4], fx.Int32),
|
||||
rsrc(c["part"]),
|
||||
off + 4 * hlf,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
publish(c, c["srdy"], t * nchunk + ch)
|
||||
stamp(c, 7)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- combine
|
||||
|
||||
COMB_HEADS = WAVES # heads a combine unit: a wave each
|
||||
|
||||
|
||||
@traced
|
||||
def stage_combine(c, unit):
|
||||
"""(token, 8 heads) ``unit``, a wave a head, 8 dims a lane: the chunks
|
||||
merged with the sink folded in, bf16, inverse RoPE, MXFP8 of each 32 -> XO,
|
||||
published (a flag a (token, head group))."""
|
||||
lane, wave, d, s = c["lane"], c["wave"], c["d"], c["S"]
|
||||
a = c["args"]
|
||||
hg_n = d.heads // COMB_HEADS
|
||||
t = unit // hg_n
|
||||
hg = unit % hg_n
|
||||
h = hg * COMB_HEADS + wave
|
||||
nchunk = c["nchunk"]
|
||||
ck = chunk_keys(d, s)
|
||||
kv_len = ld_i32(a["klen"], t)
|
||||
nlive = (kv_len + ck - 1) // ck
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
sink = ld_f32(a["sink"], h) * fx.Float32(LOG2E)
|
||||
d0 = lane * 8 # this lane's 8 dims
|
||||
rope_lane = d0 >= NOPE
|
||||
jj = fx.max((d0 - NOPE) // 2, 0) # 4 pairs from here
|
||||
cs = [
|
||||
rope_lane.select(ld_f32(a["cos_sin"], pos * ROPE + jj + i), fx.Float32(1.0))
|
||||
for i in range(4)
|
||||
]
|
||||
sn = [
|
||||
rope_lane.select(
|
||||
ld_f32(a["cos_sin"], pos * ROPE + HALF + jj + i), fx.Float32(0.0)
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
await_flags(c, c["srdy"], nlive, lambda u: t * nchunk + u)
|
||||
m = fx.Float32(NEG)
|
||||
ms, ls, vs = [], [], []
|
||||
for cc in range_constexpr(nchunk):
|
||||
live = cc < nlive
|
||||
stv = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(c["pstat"]),
|
||||
((t * nchunk + cc) * d.heads + h) * 2,
|
||||
vec_width=2,
|
||||
dtype=T.f32,
|
||||
)
|
||||
)
|
||||
pv = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(c["part"]),
|
||||
(((t * nchunk + cc) * d.heads + h) * HEAD_DIM + d0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
)
|
||||
mc = fx.Float32(live.select(stv[0], fx.Float32(NEG)))
|
||||
ms.append(mc)
|
||||
ls.append(fx.Float32(live.select(stv[1], fx.Float32(0.0))))
|
||||
vals = []
|
||||
for e in range_constexpr(4):
|
||||
vals += [bf_lo(pv[e]), bf_hi(pv[e])]
|
||||
vs.append([fx.Float32(live.select(v, fx.Float32(0.0))) for v in vals])
|
||||
m = fx.max(m, mc)
|
||||
m_final = fx.max(m, sink)
|
||||
lsum = hw_exp2(sink - m_final)
|
||||
tot = [fx.Float32(0.0)] * 8
|
||||
for cc in range_constexpr(nchunk):
|
||||
wgt = hw_exp2(ms[cc] - m_final)
|
||||
lsum = lsum + ls[cc] * wgt
|
||||
tot = [tot[e] + vs[cc][e] * wgt for e in range(8)]
|
||||
inv = fx.Float32(1.0) / lsum
|
||||
o = [bf16_round(tot[e] * inv) for e in range(8)]
|
||||
# inverse GPT-J RoPE of each pair: e' = e cos + o sin, o' = o cos - e sin
|
||||
y = []
|
||||
for i in range_constexpr(4):
|
||||
e, od = o[2 * i], o[2 * i + 1]
|
||||
y += [e * cs[i] + od * sn[i], od * cs[i] - e * sn[i]]
|
||||
amax = abs(y[0])
|
||||
for v in y[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max) # 32 dims = 4 lanes
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(y[i] * mul) for i in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(y[4 + i] * mul) for i in range(4)])
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["xo"]),
|
||||
t * (d.q_rows // 4) + (h * HEAD_DIM + d0) // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["xos"]),
|
||||
t * (d.q_rows // 32) + (h * HEAD_DIM + d0) // 32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
publish(c, c["crdy"], t * hg_n + hg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wo_a / wo_b
|
||||
|
||||
|
||||
@traced
|
||||
def stage_woa(c, task, ops, t0, n):
|
||||
"""Rows 32 task .. of wo_a -> bf16 -> vLLM's MXFP8 of the 32 -> X8B."""
|
||||
tid, red, d, lane = c["tid"], c["red"], c["d"], c["lane"]
|
||||
split = WOA_SPLIT_K
|
||||
gemv_mfmas(c, d.group_k, c["xl"], c["xsl"], red, ops, split, tiled=True, rows=n)
|
||||
gpu.barrier()
|
||||
mine = tid < WOA_ROWS * n
|
||||
tl = fx.min(tid // WOA_ROWS, n - 1)
|
||||
t = plus(t0, tl)
|
||||
row = tid % WOA_ROWS
|
||||
tile = row // ROWS
|
||||
y = bf16_round(
|
||||
row_sum(red, row % ROWS, tl, [tile * split + w for w in range(split)])
|
||||
)
|
||||
amax = butterfly(abs(y), (1, 2, 4, 8, 16), fx.max)
|
||||
code = mx_code(amax)
|
||||
q = clamp_fp8(y * mx_mul(code))
|
||||
w = fp8_pack4(
|
||||
q,
|
||||
*[
|
||||
lane_gather(q.bitcast(fx.Int32), fx.min(lane + k, 63)).bitcast(fx.Float32)
|
||||
for k in (1, 2, 3)
|
||||
],
|
||||
)
|
||||
if mine & (row % 4 == 0):
|
||||
bo.buffer_store(
|
||||
w,
|
||||
rsrc(c["x8b"]),
|
||||
t * (d.o_rows // 4) + (task * WOA_ROWS + row) // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if mine & (row == 0):
|
||||
bo.buffer_store(
|
||||
code, rsrc(c["x8bs"]), t * (d.o_rows // 32) + task, cache_modifier=CM_DEV
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
def woa_loads(c, task):
|
||||
a = c["args"]
|
||||
return gemv_loads(
|
||||
c,
|
||||
a["woa"],
|
||||
a["woa_s"],
|
||||
c["d"].group_k,
|
||||
task * WOA_ROWS,
|
||||
WOA_SPLIT_K,
|
||||
tiled=True,
|
||||
)
|
||||
|
||||
|
||||
def wob_loads(c, task, pred=None):
|
||||
a = c["args"]
|
||||
return gemv_loads(c, a["wob"], a["wob_s"], c["d"].o_rows, task * ROWS, pred=pred)
|
||||
|
||||
|
||||
def _wob_store(a, t, col, v0, v1):
|
||||
word = (
|
||||
fx.Vector.from_elements([v0, v1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
bo.buffer_store(word, rsrc(a["out"]), (t * a["out_stride"] + col) // 2)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wob(c, task, ops, task2, ops2, has2, t0, n, mark=False):
|
||||
"""Rows 16 task .. (and 16 task2 .., ``has2``) of wo_b -> bf16 -> out, or
|
||||
(``c["wob_out"]``, the mono layer) the caller's hook with each bf16 pair."""
|
||||
tid, red = c["tid"], c["red"]
|
||||
o_rows = c["d"].o_rows
|
||||
a = c["args"]
|
||||
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red, ops, rows=n)
|
||||
gemv_mfmas(c, o_rows, c["xl"], c["xsl"], red + RED, ops2, rows=n)
|
||||
gpu.barrier()
|
||||
if const_expr(mark):
|
||||
stamp(c, 15)
|
||||
per = ROWS * n // 2 # bf16 pairs a task
|
||||
k = tid // per
|
||||
emit = c.get("wob_out") or (lambda t, col, v0, v1: _wob_store(a, t, col, v0, v1))
|
||||
if (tid < 2 * per) & ((k == 0) | has2):
|
||||
u = tid % per
|
||||
rp = u % (ROWS // 2)
|
||||
tl = u // (ROWS // 2)
|
||||
t = plus(t0, tl)
|
||||
rk = red + k * RED
|
||||
col = (k == 0).select(task, task2) * ROWS + 2 * rp
|
||||
v0 = row_sum(rk, 2 * rp, tl)
|
||||
v1 = row_sum(rk, 2 * rp + 1, tl)
|
||||
emit(t, col, v0, v1)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kernel
|
||||
|
||||
|
||||
def back_plan(key):
|
||||
"""(nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob) of a build."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
ck = chunk_keys(d, s)
|
||||
nchunk = cdiv(KEYS, ck)
|
||||
n_split = s * nchunk
|
||||
n_comb = s * (d.heads // COMB_HEADS)
|
||||
COMB0 = n_split % BLOCKS
|
||||
assert d.woa_tasks <= BLOCKS
|
||||
assert BLOCKS < d.wob_tasks <= 2 * BLOCKS
|
||||
# the GEMVs past the attention units, so CTAs without attention work have
|
||||
# their weights in flight from the start
|
||||
WOA0 = (n_split + n_comb) % BLOCKS
|
||||
WOB0 = (WOA0 + d.woa_tasks) % BLOCKS
|
||||
# wo_b's weights before the wo_a stage only while the wo_a CTAs have no
|
||||
# attention work (loads complete in issue order: else they would hold up
|
||||
# wo_a's activation rows)
|
||||
early_wob = n_split + n_comb + d.woa_tasks <= BLOCKS
|
||||
return nchunk, n_split, n_comb, COMB0, WOA0, WOB0, early_wob
|
||||
|
||||
|
||||
def back_lds_members(s: int, d: Dims) -> dict:
|
||||
"""K2's attention LDS structs: the split stage's and the GEMVs' (a union's
|
||||
members: one at a time)."""
|
||||
rows = tile_rows(s)
|
||||
ck = chunk_keys(d, s)
|
||||
|
||||
@fx.struct
|
||||
class SplitLds:
|
||||
q: fx.Array[fx.Int32, d.heads * Q_ROW, 16]
|
||||
raw: fx.Array[fx.Int32, ck * REC // 4, 16]
|
||||
p: fx.Array[fx.Int32, d.heads * p_row(d, s), 16]
|
||||
st: fx.Array[fx.Float32, 2 * d.heads * (WAVES // head_groups(d, s)), 16]
|
||||
|
||||
@fx.struct
|
||||
class GemvLds:
|
||||
red: fx.Array[fx.Float32, 2 * RED, 16]
|
||||
xl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 4), 16]
|
||||
xsl: fx.Array[fx.Int32, rows * lds_row(d.group_k // 32), 16]
|
||||
|
||||
return {"split": SplitLds, "gemv": GemvLds}
|
||||
|
||||
|
||||
def back_lds(s: int, d: Dims):
|
||||
"""K2's attention LDS: the union of ``back_lds_members``."""
|
||||
m = back_lds_members(s, d)
|
||||
SplitLds, GemvLds = m["split"], m["gemv"]
|
||||
|
||||
@fx.union
|
||||
class BackLds:
|
||||
split: SplitLds # type: ignore[valid-type]
|
||||
gemv: GemvLds # type: ignore[valid-type]
|
||||
|
||||
return BackLds
|
||||
|
||||
|
||||
def back_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
|
||||
"""The attention back's context: LDS views (``lds``: an allocated
|
||||
``back_lds``, or a dict of the members' views), mailbox (tag ``tag``),
|
||||
scratch regions, arguments by name."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
tid = fx.thread_idx.x
|
||||
if isinstance(lds, dict):
|
||||
spl, gml = lds["split"], lds["gemv"]
|
||||
else:
|
||||
spl, gml = lds.split.peek(), lds.gemv.peek()
|
||||
c = {
|
||||
"S": s,
|
||||
"tid": tid,
|
||||
"bid": fx.block_idx.x,
|
||||
"lane": tid % 64,
|
||||
"wave": tid // 64,
|
||||
"red": gml.red.ptr,
|
||||
"xl": gml.xl.ptr,
|
||||
"xsl": gml.xsl.ptr,
|
||||
"d": d,
|
||||
"ratio": key.ratio,
|
||||
"mb": Mailbox(tag),
|
||||
"nchunk": back_plan(key)[0],
|
||||
"sl": {"q": spl.q.ptr, "raw": spl.raw.ptr, "p": spl.p.ptr, "st": spl.st.ptr},
|
||||
"args": args,
|
||||
}
|
||||
for region, (off, _) in (
|
||||
layout or back_scratch(s, d, start=front_end(s, d))
|
||||
).items():
|
||||
c[region] = scratch + fx.Int64(off)
|
||||
return c
|
||||
|
||||
|
||||
EPOCH_MARKS = 4 # word offset of the per-CTA marks in the epoch buffer
|
||||
|
||||
|
||||
@traced
|
||||
def epoch_begin(c, epoch, ep):
|
||||
"""This CTA has read the launch pair's epoch: its own mark word (no
|
||||
contended atomic) := ep + 1, which ``epoch_end`` checks."""
|
||||
if c["tid"] == 0:
|
||||
bo.buffer_store(
|
||||
ep + 1, rsrc(epoch), EPOCH_MARKS + c["bid"], cache_modifier=CM_DEV
|
||||
)
|
||||
|
||||
|
||||
@traced
|
||||
def epoch_end(c, epoch, ep, reset=None):
|
||||
"""CTA 0, once every CTA marked this epoch (thread u checks CTA u's mark):
|
||||
``reset()`` (state later pairs must find cleared), then the epoch moved on.
|
||||
The next pair's hand-offs carry a tag no earlier pair wrote; later launches'
|
||||
kernels start after this one ends."""
|
||||
if c["bid"] == 0:
|
||||
tid = c["tid"]
|
||||
if tid < BLOCKS:
|
||||
r = rsrc(epoch)
|
||||
|
||||
def load_mark():
|
||||
return fx.Int32(
|
||||
bo.buffer_load(
|
||||
r,
|
||||
EPOCH_MARKS + tid,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
)
|
||||
|
||||
_ = spin_until(load_mark, lambda v: v != ep + 1)
|
||||
gpu.barrier()
|
||||
if tid == 0:
|
||||
if const_expr(reset is not None):
|
||||
reset()
|
||||
gstore(epoch, ep + 1, words=1)
|
||||
|
||||
|
||||
@traced
|
||||
def run_back(c, key, bid):
|
||||
"""The attention back's stages on this CTA: split, combine, wo_a, wo_b."""
|
||||
s = key.tokens
|
||||
dd = Dims(key.tp)
|
||||
# plain ints: a Python object read inside traced control flow is carried
|
||||
woa_tasks, groups, heads, group_k = dd.woa_tasks, dd.groups, dd.heads, dd.group_k
|
||||
q_rows, o_rows, wob_tasks = dd.q_rows, dd.o_rows, dd.wob_tasks
|
||||
a = c["args"]
|
||||
tiles = token_tiles(s)
|
||||
_, n_split, n_comb, COMB0, WOA0, WOB0, early_wob = back_plan(key)
|
||||
stamp(c, 0)
|
||||
for unit in range(first_task(bid, 0), n_split, BLOCKS):
|
||||
stage_split(c, unit)
|
||||
stamp(c, 1)
|
||||
for unit in range(first_task(bid, COMB0), n_comb, BLOCKS):
|
||||
stage_combine(c, unit)
|
||||
gpu.barrier()
|
||||
stamp(c, 2)
|
||||
# the GEMV weights, every CTA right after its attention work: its wo_a
|
||||
# task (if any), then its one or two wo_b tasks
|
||||
woa_task = first_task(bid, WOA0)
|
||||
has_woa = woa_task < woa_tasks
|
||||
woa_t = fx.min(woa_task, woa_tasks - 1)
|
||||
ops = gemv_loads(
|
||||
c,
|
||||
a["woa"],
|
||||
a["woa_s"],
|
||||
group_k,
|
||||
woa_t * WOA_ROWS,
|
||||
WOA_SPLIT_K,
|
||||
tiled=True,
|
||||
pred=has_woa,
|
||||
)
|
||||
wob_task = first_task(bid, WOB0)
|
||||
wob_second = wob_task + BLOCKS
|
||||
has_second = wob_second < wob_tasks
|
||||
|
||||
def wob_weights():
|
||||
return (
|
||||
wob_loads(c, wob_task),
|
||||
wob_loads(c, fx.min(wob_second, wob_tasks - 1), pred=has_second),
|
||||
)
|
||||
|
||||
if const_expr(early_wob):
|
||||
ops1, ops2 = wob_weights()
|
||||
if const_expr(c.get("tl") is not None):
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
stamp(c, 8)
|
||||
# wo_a: every token tile of its group's XO rows, the next tile's in flight
|
||||
if has_woa:
|
||||
grp = woa_t // (woa_tasks // groups)
|
||||
await_flags(c, c["crdy"], s, lambda u: u * (heads // COMB_HEADS) + grp)
|
||||
stamp(c, 9)
|
||||
xo = (
|
||||
c["xo"],
|
||||
c["xos"],
|
||||
q_rows // 4,
|
||||
grp * group_k // 4,
|
||||
group_k // 4,
|
||||
q_rows // 32,
|
||||
grp * group_k // 32,
|
||||
group_k // 32,
|
||||
)
|
||||
regs = rows_loads(c, *xo, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, group_k // 4, group_k // 32, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *xo, *tiles[ti + 1])
|
||||
stage_woa(c, woa_t, ops, t0, n)
|
||||
stamp(c, 10)
|
||||
publish(c, c["ardy"], woa_t)
|
||||
stamp(c, 3)
|
||||
# wo_b
|
||||
if const_expr(not early_wob):
|
||||
ops1, ops2 = wob_weights()
|
||||
stamp(c, 11)
|
||||
await_flags(c, c["ardy"], woa_tasks, lambda u: u)
|
||||
stamp(c, 12)
|
||||
x8b = (
|
||||
c["x8b"],
|
||||
c["x8bs"],
|
||||
o_rows // 4,
|
||||
0,
|
||||
o_rows // 4,
|
||||
o_rows // 32,
|
||||
0,
|
||||
o_rows // 32,
|
||||
)
|
||||
regs = rows_loads(c, *x8b, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, o_rows // 4, o_rows // 32, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti == 0):
|
||||
stamp(c, 14)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *x8b, *tiles[ti + 1])
|
||||
stage_wob(
|
||||
c,
|
||||
wob_task,
|
||||
ops1,
|
||||
fx.min(wob_second, wob_tasks - 1),
|
||||
ops2,
|
||||
has_second,
|
||||
t0,
|
||||
n,
|
||||
mark=ti == 0,
|
||||
)
|
||||
if const_expr(ti == 0):
|
||||
stamp(c, 13)
|
||||
stamp(c, 4)
|
||||
@@ -1,504 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Device primitives of the DeepSeek-V4.1 attention mega kernel.
|
||||
|
||||
Wave reductions and shuffles, hardware math, bf16 / fp8 / MX packing, the
|
||||
scaled MFMA, buffer and 64-bit global accesses, and the tagged-pair mailbox.
|
||||
Adapted from ATOM's mono decode framework (``atom/mono/device``), itself ported
|
||||
from the FlyDSL GLM-5 layer kernel.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
|
||||
from aiter.ops.flydsl.kernels.kernels_common import kernel_signature
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import llvm
|
||||
from flydsl.compiler.ast_rewriter import ASTRewriter
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
# The execution model: one CTA a CU of an MI355X, all resident.
|
||||
BLOCKS = 256
|
||||
THREADS = 512
|
||||
WAVES = THREADS // 64
|
||||
|
||||
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16, NT = 2. SC1 alone is
|
||||
# device scope (past the per-XCD L2s); NT streams data read once.
|
||||
CM_DEV = 16
|
||||
CM_NT = 2
|
||||
|
||||
FP8_MAX = 448.0
|
||||
FP8, FP4 = 0, 4 # f8f6f4 MFMA operand formats
|
||||
UNIT_SCALE = 127 # E8M0 code of 1.0
|
||||
FLT_MIN = 1.1754943508222875e-38
|
||||
|
||||
POLL_MAX = 12 # mailbox pairs polled a batch
|
||||
|
||||
|
||||
def traced(fn):
|
||||
"""FlyDSL's AST rewrite for a helper, so its ``if`` / ``while`` / ``for`` on
|
||||
traced values lower to scf ops (kernels rewrite only their own body)."""
|
||||
return ASTRewriter.transform(fn)
|
||||
|
||||
|
||||
def kernel_symbol(stem, **params):
|
||||
"""Kernel symbol that profilers demangle to ``aiter::<stem>_<params>``."""
|
||||
name = f"{stem}_{kernel_signature(**params)}"
|
||||
return f"_ZN5aiter{len(name)}{name}E"
|
||||
|
||||
|
||||
def first_task(bid, base):
|
||||
"""CTA ``bid``'s first task of a stage placed from CTA ``base``, then every
|
||||
BLOCKS-th."""
|
||||
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- memory
|
||||
|
||||
|
||||
def rsrc(addr, nbytes=None):
|
||||
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
|
||||
|
||||
|
||||
def ld_i32(ptr, i, cm=0):
|
||||
return fx.Int32(
|
||||
bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.i32, cache_modifier=cm)
|
||||
)
|
||||
|
||||
|
||||
def ld_f32(ptr, i):
|
||||
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
|
||||
|
||||
|
||||
def ld_bf(ptr, i):
|
||||
return fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
)
|
||||
|
||||
|
||||
def ld_i64(ptr, i):
|
||||
"""Element i of an i64 buffer (two dwords)."""
|
||||
w = fx.Vector(bo.buffer_load(rsrc(ptr), i * 2, vec_width=2, dtype=T.i32))
|
||||
return (fx.Int64(w[1]) << 32) | fx.Int64(fx.Uint32(w[0]))
|
||||
|
||||
|
||||
def _gptr(addr):
|
||||
return llvm.IntToPtrOp(
|
||||
ir.Type.parse("!llvm.ptr<1>"), as_ir_value(fx.Int64(addr))
|
||||
).result
|
||||
|
||||
|
||||
def gload(addr, words=4, nt=False):
|
||||
"""``words`` i32 (1, 2 or 4) at a 64-bit global address: a cache past 4 GiB is
|
||||
reachable only this way (a buffer offset is 32 bits)."""
|
||||
i32 = ir.IntegerType.get_signless(32)
|
||||
ty = i32 if words == 1 else ir.VectorType.get([words], i32)
|
||||
v = llvm.LoadOp(ty, _gptr(addr), alignment=4 * words, nontemporal=nt or None).result
|
||||
return fx.Int32(v) if words == 1 else fx.Vector(v)
|
||||
|
||||
|
||||
def atomic_add_dev(addr, value):
|
||||
"""Device-scope atomic add on the global i32 at ``addr``; the old value.
|
||||
Relaxed: an acquire / release here would write back and invalidate L2."""
|
||||
return fx.Int32(
|
||||
fx.atomic_add(
|
||||
_gptr(addr),
|
||||
fx.Int32(value),
|
||||
syncscope="agent",
|
||||
ordering=fx.AtomicOrdering.Monotonic,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def gstore(addr, value, words=4):
|
||||
"""Store ``words`` i32 (an Int32 or a Vector) at a 64-bit global address."""
|
||||
v = as_ir_value(fx.Int32(value) if words == 1 else value)
|
||||
llvm.StoreOp(v, _gptr(addr), alignment=4 * words)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- math
|
||||
|
||||
|
||||
def fresh(v):
|
||||
"""``v`` through an opaque move, so the lane math built on it is computed at
|
||||
its use instead of being hoisted out of a task loop and held in registers."""
|
||||
return fx.Int32(
|
||||
llvm.inline_asm(
|
||||
T.i32,
|
||||
[fx.Int32(v).ir_value()],
|
||||
"v_mov_b32 $0, $1",
|
||||
"=v,v",
|
||||
has_side_effects=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def tile_map(tid, width):
|
||||
"""(tid // width, tid % width) for 0 <= tid < THREADS without a division:
|
||||
shifts for a power of two, else a multiply-shift exact on that range."""
|
||||
if width & (width - 1) == 0:
|
||||
sh = width.bit_length() - 1
|
||||
return tid >> sh, tid & (width - 1)
|
||||
m = -(-(1 << 20) // width)
|
||||
assert all((t * m) >> 20 == t // width for t in range(THREADS)), width
|
||||
row = (tid * m) >> 20
|
||||
return row, tid - row * width
|
||||
|
||||
|
||||
def uniform(v):
|
||||
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
|
||||
|
||||
|
||||
def _hw_f32(name, x):
|
||||
return fx.Float32(
|
||||
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
|
||||
)
|
||||
|
||||
|
||||
def hw_rsq(x):
|
||||
return _hw_f32("llvm.amdgcn.rsq.f32", x)
|
||||
|
||||
|
||||
def hw_rcp(x):
|
||||
return _hw_f32("llvm.amdgcn.rcp.f32", x)
|
||||
|
||||
|
||||
def hw_exp2(x):
|
||||
return _hw_f32("llvm.amdgcn.exp2.f32", x)
|
||||
|
||||
|
||||
def fma(a, b, c):
|
||||
return fx.Float32(fmath.fma(fx.Float32(a), fx.Float32(b), fx.Float32(c)))
|
||||
|
||||
|
||||
def div_rn(x, d, r):
|
||||
"""``x / d`` rounded to nearest (IEEE), given ``r = RN(1 / d)``."""
|
||||
q = x * r
|
||||
return fma(fma(-q, d, x), r, q)
|
||||
|
||||
|
||||
def permlane_swap(off, x, y):
|
||||
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
|
||||
pr = swap(
|
||||
llvm.StructType.get_literal([T.i32, T.i32]),
|
||||
as_ir_value(fx.Int32(x)),
|
||||
as_ir_value(fx.Int32(y)),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
|
||||
|
||||
|
||||
def lane_gather(v, src_lane):
|
||||
"""Lane ``src_lane``'s value of v (ds_bpermute)."""
|
||||
return fx.Int32(
|
||||
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
|
||||
)
|
||||
|
||||
|
||||
def lane_f32(v, src_lane):
|
||||
return lane_gather(v.bitcast(fx.Int32), src_lane).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def readlane(v, src_lane):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
r = fx.Int32(rocdl.readlane(T.i32, bits, fx.Int32(src_lane)))
|
||||
return r.bitcast(fx.Float32) if isinstance(v, fx.Float32) else r
|
||||
|
||||
|
||||
def xshfl(v, off):
|
||||
"""Value of lane ``lane ^ off`` (DPP in a row, permlane swaps across)."""
|
||||
assert off in (1, 2, 4, 8, 16, 32), off
|
||||
if off >= 16:
|
||||
return v.shuffle_xor(off, 64)
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
if off == 8:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
|
||||
elif off == 4:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
|
||||
elif off == 2:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
|
||||
else:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
|
||||
return y.bitcast(fx.Float32) if is_f else y
|
||||
|
||||
|
||||
def xred(v, off, op):
|
||||
if off < 16:
|
||||
return op(v, xshfl(v, off))
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
a, b = permlane_swap(off, x, x)
|
||||
if is_f:
|
||||
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
|
||||
return op(type(v)(a), type(v)(b))
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return a + b
|
||||
|
||||
|
||||
def butterfly(v, offsets, op=_add):
|
||||
"""Butterfly reduction over the xor ``offsets``, in that order."""
|
||||
for off in offsets:
|
||||
v = xred(v, off, op)
|
||||
return v
|
||||
|
||||
|
||||
def wave_sum(v):
|
||||
return butterfly(v, (32, 16, 8, 4, 2, 1))
|
||||
|
||||
|
||||
def wave_max(v):
|
||||
return butterfly(v, (32, 16, 8, 4, 2, 1), fx.max)
|
||||
|
||||
|
||||
def row_shr(v, lane, k):
|
||||
"""Lane - k of the same 16-lane row, else 0 (DPP row_shr)."""
|
||||
moved = lane_gather(v.bitcast(fx.Int32), fx.max(lane - k, 0)).bitcast(fx.Float32)
|
||||
return fx.Float32((lane % 16 >= k).select(moved, fx.Float32(0.0)))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- packing
|
||||
|
||||
|
||||
def bf16_round(a):
|
||||
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
|
||||
|
||||
|
||||
def bf16_pair(a, b):
|
||||
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
|
||||
return (
|
||||
fx.Vector.from_elements([a, b], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Float32)[0]
|
||||
)
|
||||
|
||||
|
||||
def bf_lo(w):
|
||||
return (w << 16).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf_hi(w):
|
||||
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def fp8_pack4(a, b, c, d):
|
||||
"""Four f32 (in range) -> one dword of four E4M3 bytes, RNE."""
|
||||
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
|
||||
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
|
||||
|
||||
|
||||
def fp8x8_bf16(w0, w1, scale):
|
||||
"""Eight E4M3 of two dwords times an f32 ``scale`` -> eight bf16 as a
|
||||
4-dword Vector (exact for a power-of-two scale in range)."""
|
||||
out = []
|
||||
for w in (w0, w1):
|
||||
for hi in (False, True):
|
||||
p = rocdl.cvt_scalef32_pk_bf16_fp8(
|
||||
T.bf16x2, as_ir_value(fx.Int32(w)), as_ir_value(fx.Float32(scale)), hi
|
||||
)
|
||||
out.append(fx.Vector(p).bitcast(fx.Int32)[0])
|
||||
return fx.Vector.from_elements(out, fx.Int32)
|
||||
|
||||
|
||||
def clamp_fp8(v):
|
||||
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
||||
|
||||
|
||||
def pow2(code):
|
||||
"""The fp32 power of two a biased exponent (1..254) stands for."""
|
||||
return (code << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def rcp_pow2(code):
|
||||
"""1 / ``pow2(code)``, exact for code in [1, 253]."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def ceil_exp(v):
|
||||
"""Biased exponent of the smallest power of two >= v (v > 0 finite, normal)."""
|
||||
bits = v.bitcast(fx.Int32)
|
||||
return ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
||||
fx.Int32(1), fx.Int32(0)
|
||||
)
|
||||
|
||||
|
||||
def mx_code(amax):
|
||||
"""VLLM's ROCm MXFP8 scale code of a 32-group: clamp(ceil(log2(amax / 448))
|
||||
+ 127, 0, 254), amax floored at FLT_MIN."""
|
||||
a = fx.max(amax, fx.Float32(FLT_MIN))
|
||||
q = div_rn(a, fx.Float32(FP8_MAX), fx.Float32(1.0 / FP8_MAX))
|
||||
return fx.min(fx.max(ceil_exp(q), fx.Int32(0)), fx.Int32(254))
|
||||
|
||||
|
||||
def mx_mul(code):
|
||||
"""2^(127 - code) as fp32, for code in [0, 254] (the quant multiplier)."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- MFMA
|
||||
|
||||
|
||||
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8):
|
||||
"""16x16x128 scaled MFMA (gfx950 f8f6f4): lane l (row l % 16) holds K bytes
|
||||
16 (l // 16) .. +16 and 64 + 16 (l // 16) .. +16 of the 128-step, and the
|
||||
E8M0 of K block l // 16."""
|
||||
return fx.Vector(
|
||||
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
||||
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, 0, sa, 0, sb]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def mfma_bf16(a, b, c):
|
||||
"""16x16x32 bf16 MFMA: lane l (row / col l % 16) holds K 8 (l // 16) .. +8."""
|
||||
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- mailbox
|
||||
|
||||
|
||||
@traced
|
||||
def _spin(load_all, pending):
|
||||
v = load_all()
|
||||
while pending(v):
|
||||
rocdl.s_sleep(1)
|
||||
v = load_all()
|
||||
return v
|
||||
|
||||
|
||||
def spin_until(load, pending):
|
||||
"""``load()`` again (s_sleep between) while ``pending(value)``; the value."""
|
||||
return _spin(load, pending)
|
||||
|
||||
|
||||
class Mailbox:
|
||||
"""Tagged-pair hand-off between the CTAs of one launch (device scope).
|
||||
|
||||
Every 32-bit value is stored next to the launch's tag in one 8-byte store; a
|
||||
consumer polls until every tag matches, so a hand-off is one round trip with
|
||||
no separate flag. The tag is ``epoch * 64 + layer + 1`` (``epoch`` a device
|
||||
counter bumped once a step): a pair from an earlier layer or step never
|
||||
carries a live tag, and nothing is cleared between launches."""
|
||||
|
||||
def __init__(self, tag):
|
||||
self.tag = tag
|
||||
|
||||
def put(self, base, i, v):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
||||
rsrc(base),
|
||||
i * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def put_bf(self, base, i, vs):
|
||||
"""Elements i .. i + len(vs) (2 or 4, i aligned) as bf16 pairs, one store."""
|
||||
words = []
|
||||
for j in range_constexpr(len(vs) // 2):
|
||||
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(base),
|
||||
i,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def put_words(self, base, i, words):
|
||||
"""Raw 32-bit words (1 or 2) at pairs i.. (fp8 words must come this way)."""
|
||||
vals = []
|
||||
for w in words:
|
||||
vals += [fx.Int32(w), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(vals, fx.Int32),
|
||||
rsrc(base),
|
||||
i * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
def poll(self, specs, batch=POLL_MAX):
|
||||
"""Batched poll of ``specs`` = [(base, pair index, npairs in {1, 2})]: all
|
||||
re-loaded while any tag is stale. Returns the value words."""
|
||||
if const_expr(len(specs) == 0):
|
||||
return []
|
||||
if const_expr(len(specs) > batch):
|
||||
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
||||
tag = self.tag
|
||||
|
||||
def load_all():
|
||||
words = []
|
||||
for b, i, n in specs:
|
||||
w = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(b),
|
||||
fx.Int32(i) * 2,
|
||||
vec_width=2 * n,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
)
|
||||
words += [w[e] for e in range(2 * n)]
|
||||
return fx.Vector.from_elements(words, fx.Int32)
|
||||
|
||||
nw = sum(2 * n for _, _, n in specs)
|
||||
|
||||
def pending(v):
|
||||
bad = v[1] != tag
|
||||
for e in range_constexpr(3, nw, 2):
|
||||
bad = bad | (v[e] != tag)
|
||||
return bad
|
||||
|
||||
v = _spin(load_all, pending)
|
||||
outs, e = [], 0
|
||||
for _, _, n in specs:
|
||||
outs.append([v[e + 2 * q] for q in range(n)])
|
||||
e += 2 * n
|
||||
return outs
|
||||
|
||||
|
||||
LOAD_BATCH = 4
|
||||
|
||||
|
||||
@traced
|
||||
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
|
||||
"""A CTA's strided loop over v = tid + THREADS i < n: ``batch`` rounds' loads
|
||||
issued before their uses; a round past n loads element n - 1, uses nothing."""
|
||||
rounds = -(-n // THREADS)
|
||||
for b0 in range_constexpr(0, rounds, batch):
|
||||
idx = range(b0, min(b0 + batch, rounds))
|
||||
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
|
||||
for i, g in zip(idx, got):
|
||||
v = tid + THREADS * i
|
||||
if v < n:
|
||||
use(v, g)
|
||||
|
||||
|
||||
def barrier():
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------- timeline
|
||||
|
||||
|
||||
def memrealtime():
|
||||
"""s_memrealtime: the 100 MHz device clock."""
|
||||
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
|
||||
|
||||
|
||||
@traced
|
||||
def stamp(c, point):
|
||||
"""A timeline build's stamp: thread 0 of the CTA stores the clock at
|
||||
``point`` (``c["tl"]`` [BLOCKS, points] i64; None: not a timeline build)."""
|
||||
if const_expr(c.get("tl") is None):
|
||||
return
|
||||
if c["tid"] == 0:
|
||||
t = memrealtime()
|
||||
w = fx.Vector.from_elements([fx.Int32(t), fx.Int32(t >> 32)], fx.Int32)
|
||||
bo.buffer_store(w, rsrc(c["tl"]), (c["bid"] * c["tl_points"] + point) * 2)
|
||||
@@ -1,680 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K1 ``front``: one DeepSeek-V4.1 attention layer from its normed input row to
|
||||
the rotated query and the step's KV rows in the sliding-window cache.
|
||||
|
||||
One launch a layer and rank, ``BLOCKS`` CTAs x ``THREADS`` threads, S rows.
|
||||
Every CTA issues its weight loads (its wqkv part and its wq_b tasks) at the
|
||||
kernel start, before it waits on anything; the stages:
|
||||
|
||||
xq (S > 16: S x 2 K halves, on the 32 CTAs without a wqkv part): the
|
||||
bf16 input's half -> vLLM's MXFP8 -> X8 (published, a flag each)
|
||||
kt (S): the token's attention keys for K2 -- top-k compressed slots
|
||||
through the kv source's block table, then window slots -> KT
|
||||
wqkv_a (112 row tiles x 2 K halves, 16 rows): the input's MXFP8 half --
|
||||
quantized in the CTA (S <= 16) or X8's -- a token tile at a time
|
||||
(the next tile's rows in flight) times wqkv's -> fp32 partials ->
|
||||
QKVP
|
||||
qkv (S): the halves summed -> bf16; q_norm -> bf16 -> MXFP8 -> QX8
|
||||
(published, a flag a token); kv_norm -> bf16 -> GPT-J RoPE -> bf16
|
||||
-> the fp8_ds_mla record at the token's slot
|
||||
wq_b (H x 512 / 16, 16 rows): QX8 times wq_b -> bf16 -> GPT-J RoPE of
|
||||
each head's dims 448.. -> q (global, bf16)
|
||||
|
||||
Each elementwise step follows vLLM's ROCm path (``mxfp8_quantize``,
|
||||
``FusedQKVRMSNorm``, ``fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert``);
|
||||
the GEMV reductions differ from the Triton GEMMs' only in their order.
|
||||
Adapted from ATOM's V4.1 ``attn_pre`` (K1), less its mHC seam.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from .device import (
|
||||
BLOCKS,
|
||||
CM_DEV,
|
||||
THREADS,
|
||||
Mailbox,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
ceil_exp,
|
||||
clamp_fp8,
|
||||
div_rn,
|
||||
first_task,
|
||||
fp8_pack4,
|
||||
fresh,
|
||||
gstore,
|
||||
hw_rsq,
|
||||
ld_f32,
|
||||
ld_i32,
|
||||
mx_code,
|
||||
mx_mul,
|
||||
rsrc,
|
||||
stamp,
|
||||
tile_map,
|
||||
traced,
|
||||
xshfl,
|
||||
)
|
||||
from .gemv import RED, gemv_loads, gemv_mfmas, lds_row, row_sum, rows_loads, rows_store
|
||||
from .plan import (
|
||||
DATA,
|
||||
EPS,
|
||||
HALF,
|
||||
HEAD_DIM,
|
||||
HIDDEN,
|
||||
KEYS,
|
||||
KV_DIM,
|
||||
NOPE,
|
||||
Q_RANK,
|
||||
QKV_ROWS,
|
||||
ROPE,
|
||||
ROWS,
|
||||
TILE,
|
||||
TOPK,
|
||||
WINDOW,
|
||||
WQKV_KPARTS,
|
||||
WQKV_TASKS,
|
||||
Dims,
|
||||
front_scratch,
|
||||
tile_rows,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
_STREAM = fx.Stream(None)
|
||||
X_WORDS = HIDDEN // 4
|
||||
X_GROUPS = HIDDEN // 32
|
||||
QX_WORDS = Q_RANK // 4
|
||||
KSTEPS = HIDDEN // 128 // WQKV_KPARTS # a wqkv part's 128-steps
|
||||
KWORDS = KSTEPS * 32
|
||||
KGROUPS = KSTEPS * 4
|
||||
XH = HIDDEN // WQKV_KPARTS # a K half's values
|
||||
XH_COLS = XH // 8 // 2 # 16 B items (8 values) a half of a K half
|
||||
XH_RPR = THREADS // XH_COLS # half-rows a round
|
||||
XQ_ITEMS = XH // 8 # an xq unit's 16 B items (one a thread)
|
||||
FRONT_POINTS = 6 # start, kt done, wqkv done, qkv done, wq_b done, (spare)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FrontBuild:
|
||||
tokens: int
|
||||
tp: int
|
||||
ratio: int # the layer's compress ratio (0: no top-k keys)
|
||||
timeline: bool = False
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
@traced
|
||||
def publish(c, region, i):
|
||||
"""This CTA's stores visible device-wide, then the flag pair (region, i)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if c["tid"] == 0:
|
||||
c["mb"].put(region, i, fx.Int32(1))
|
||||
|
||||
|
||||
@traced
|
||||
def await_flags(c, region, n, index):
|
||||
"""Thread u < n polls flag ``index(u)``; then the CTA proceeds."""
|
||||
tid = c["tid"]
|
||||
if tid < n:
|
||||
c["mb"].poll([(region, index(fx.min(tid, n - 1)), 1)])
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- xq
|
||||
|
||||
|
||||
def xq_loads(c, unit, pred):
|
||||
"""(token, K half) ``unit``'s bf16 input: its load only, 8 values a thread
|
||||
(threads past the half mirror its last item)."""
|
||||
a = c["args"]
|
||||
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
|
||||
col = fx.min(c["tid"], XQ_ITEMS - 1)
|
||||
e = (unit >> 1) * a["x_stride"] + (unit & 1) * XH + col * 8
|
||||
return fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_xq(c, unit, w, live):
|
||||
"""``xq_loads``' values -> vLLM's MXFP8 -> X8 words / X8S codes of the
|
||||
unit (published by the caller)."""
|
||||
tid, lane = c["tid"], c["lane"]
|
||||
vals = []
|
||||
for d in range_constexpr(4):
|
||||
vals += [bf_lo(w[d]), bf_hi(w[d])]
|
||||
amax = abs(vals[0])
|
||||
for v in vals[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max)
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
|
||||
t = unit >> 1
|
||||
part = unit & 1
|
||||
if live & (tid < XQ_ITEMS):
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["x8"]),
|
||||
t * X_WORDS + part * KWORDS + tid * 2,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["x8s"]),
|
||||
t * X_GROUPS + part * KGROUPS + (tid >> 2),
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kt
|
||||
|
||||
|
||||
@traced
|
||||
def stage_kt(c, t):
|
||||
"""Token t's keys for K2: ``KT[t][k] = slot | 1 << 31`` for its top-k compressed
|
||||
rows (k < ntopk), the window slot for k in [ntopk, kv_len), -1 for an
|
||||
invalid or absent key; KLEN[t] = kv_len (0 for a pad row), KLEN[S + t] =
|
||||
ntopk."""
|
||||
tid = c["tid"]
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
# device scope where this launch reads the table (K2's)
|
||||
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
|
||||
slot = ld_i32(a["slot"], 2 * t)
|
||||
live = slot >= 0
|
||||
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
|
||||
if const_expr(ratio > 0):
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
ntopk = live.select(fx.min((pos + 1) // ratio, fx.Int32(TOPK)), fx.Int32(0))
|
||||
req = ld_i32(a["t2r"], t)
|
||||
else:
|
||||
ntopk = fx.Int32(0)
|
||||
kv_len = ntopk + nswa
|
||||
for i in range_constexpr((KEYS + THREADS - 1) // THREADS):
|
||||
k = fresh(tid) + THREADS * i
|
||||
sidx = fx.min(fx.max(k - ntopk, 0), WINDOW - 1)
|
||||
s_slot = ld_i32(a["swa_idx"], t * WINDOW + sidx)
|
||||
val = (k < kv_len).select(s_slot, fx.Int32(-1))
|
||||
if const_expr(ratio > 0):
|
||||
local = ld_i32(a["topk"], t * TOPK + fx.min(k, TOPK - 1))
|
||||
lc = fx.max(local, 0)
|
||||
e = a["comp_block"]
|
||||
blk = ld_i32(a["comp_bt"], req * a["bt_stride"] + lc // e)
|
||||
c_slot = (blk * e + lc % e) | fx.Int32(-2147483648)
|
||||
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
|
||||
val = (k < ntopk).select(c_val, val)
|
||||
if k < KEYS:
|
||||
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
|
||||
if tid == 0:
|
||||
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
|
||||
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wqkv_a
|
||||
|
||||
|
||||
def x_half_loads(c, part, pred, t0, n):
|
||||
"""Tokens t0 .. t0 + n's bf16 input, K half ``part``: its loads only (16 B a
|
||||
lane-item, 8 values; a thread one column of [2 n half-rows, 160]), masked
|
||||
off on CTAs without a wqkv part."""
|
||||
a = c["args"]
|
||||
xr = rsrc(a["x"], pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0)))
|
||||
hr0, col = tile_map(c["tid"], XH_COLS)
|
||||
e0 = part * XH + col * 8
|
||||
out = []
|
||||
for i in range((2 * n + XH_RPR - 1) // XH_RPR):
|
||||
hr = fx.min(hr0 + XH_RPR * i, 2 * n - 1)
|
||||
e = (t0 + (hr >> 1)) * a["x_stride"] + (hr & 1) * (XH // 2) + e0
|
||||
out.append(fx.Vector(bo.buffer_load(xr, e >> 1, vec_width=4, dtype=T.i32)))
|
||||
return out
|
||||
|
||||
|
||||
@traced
|
||||
def quant_x_half(c, regs, n):
|
||||
"""``x_half_loads``' values -> vLLM's MXFP8 rows of the K half in LDS (a
|
||||
32-group = 4 lanes)."""
|
||||
tid, lane = c["tid"], c["lane"]
|
||||
x_row, s_row = lds_row(KWORDS), lds_row(KGROUPS)
|
||||
hr0, col = tile_map(tid, XH_COLS)
|
||||
act = tid < XH_RPR * XH_COLS
|
||||
for i in range_constexpr(len(regs)):
|
||||
hr = hr0 + XH_RPR * i
|
||||
w = regs[i]
|
||||
vals = []
|
||||
for d in range_constexpr(4):
|
||||
vals += [bf_lo(w[d]), bf_hi(w[d])]
|
||||
amax = abs(vals[0])
|
||||
for v in vals[1:]:
|
||||
amax = fx.max(amax, abs(v))
|
||||
amax = butterfly(amax, (1, 2), fx.max)
|
||||
code = mx_code(amax)
|
||||
mul = mx_mul(code)
|
||||
w0 = fp8_pack4(*[clamp_fp8(vals[q] * mul) for q in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(vals[4 + q] * mul) for q in range(4)])
|
||||
tok = hr >> 1
|
||||
half = hr & 1
|
||||
if act & (hr < 2 * n):
|
||||
fx.ptr_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
c["xl"] + (tok * x_row + half * (KWORDS // 2) + col * 2),
|
||||
)
|
||||
if lane % 4 == 0:
|
||||
fx.ptr_store(
|
||||
code, c["xsl"] + (tok * s_row + half * (KGROUPS // 2) + (col >> 2))
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wqkv(c, task, part, ops, t0, n, live):
|
||||
"""Rows 16 task .. of wqkv over K half ``part`` against tokens t0 .. t0 + n
|
||||
-> fp32 partials QKVP[part] (``live``: a CTA with a wqkv part)."""
|
||||
tid, red = c["tid"], c["red"]
|
||||
gemv_mfmas(c, HIDDEN, c["xl"], c["xsl"], red, ops, rows=n, nsteps=KSTEPS)
|
||||
gpu.barrier()
|
||||
if (tid < ROWS * n) & live:
|
||||
r = tid % ROWS
|
||||
tl = tid // ROWS
|
||||
t = plus(t0, tl)
|
||||
v = row_sum(red, r, tl)
|
||||
c["mb"].put(c["qkvp"], (part * c["S"] + t) * QKV_ROWS + task * ROWS + r, v)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- q / kv norms
|
||||
|
||||
|
||||
@traced
|
||||
def stage_qkv(c, t):
|
||||
"""Token t: the K halves summed -> bf16; q_norm -> bf16 -> MXFP8 (QX8);
|
||||
kv_norm -> bf16 -> RoPE -> bf16 -> the fp8_ds_mla record at
|
||||
slot_mapping[t]. 256 threads (8 q columns and 2 kv columns each); threads
|
||||
256.. mirror thread 255 and store nothing."""
|
||||
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
|
||||
a, s = c["args"], c["S"]
|
||||
tt = fx.min(tid, 255)
|
||||
mine = tid < 256
|
||||
qc = [fx.min(tt * 8 + j, Q_RANK - 1) for j in range(8)]
|
||||
# independent loads first: they overlap the partials' wait
|
||||
qw = [c["ldbf"](a["qn_w"], qc[j]) for j in range(8)]
|
||||
kw = [c["ldbf"](a["kvn_w"], tt * 2 + j) for j in range(2)]
|
||||
pos = ld_i32(a["pos"], 2 * t)
|
||||
slot = ld_i32(a["slot"], 2 * t)
|
||||
jr = tt - NOPE // 2
|
||||
is_rope = jr >= 0
|
||||
jj = fx.max(jr, 0)
|
||||
cs = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + jj), fx.Float32(1.0))
|
||||
sn = is_rope.select(ld_f32(a["cos_sin"], pos * ROPE + HALF + jj), fx.Float32(0.0))
|
||||
specs = []
|
||||
for p in range_constexpr(2):
|
||||
specs += [(c["qkvp"], (p * s + t) * QKV_ROWS + qc[j], 1) for j in range(8)]
|
||||
specs += [
|
||||
(c["qkvp"], (p * s + t) * QKV_ROWS + Q_RANK + tt * 2 + j, 1)
|
||||
for j in range(2)
|
||||
]
|
||||
got = c["mb"].poll(specs)
|
||||
vals = [
|
||||
bf16_round(got[j][0].bitcast(fx.Float32) + got[10 + j][0].bitcast(fx.Float32))
|
||||
for j in range(10)
|
||||
]
|
||||
xq = [((tt * 8 + j) < Q_RANK).select(vals[j], fx.Float32(0.0)) for j in range(8)]
|
||||
xk = vals[8:10]
|
||||
ssq = xq[0] * xq[0]
|
||||
for j in range_constexpr(1, 8):
|
||||
ssq = ssq + xq[j] * xq[j]
|
||||
ssq = butterfly(ssq, (32, 16, 8, 4, 2, 1))
|
||||
ssk = butterfly(xk[0] * xk[0] + xk[1] * xk[1], (32, 16, 8, 4, 2, 1))
|
||||
if (lane == 0) & mine:
|
||||
fx.ptr_store(ssq, red + wave)
|
||||
fx.ptr_store(ssk, red + 4 + wave)
|
||||
gpu.barrier()
|
||||
tq = (fx.ptr_load(red + 0) + fx.ptr_load(red + 1)) + (
|
||||
fx.ptr_load(red + 2) + fx.ptr_load(red + 3)
|
||||
)
|
||||
tk = (fx.ptr_load(red + 4) + fx.ptr_load(red + 5)) + (
|
||||
fx.ptr_load(red + 6) + fx.ptr_load(red + 7)
|
||||
)
|
||||
rq = hw_rsq(
|
||||
div_rn(tq, fx.Float32(float(Q_RANK)), fx.Float32(1.0 / Q_RANK))
|
||||
+ fx.Float32(EPS)
|
||||
)
|
||||
rk = hw_rsq(
|
||||
div_rn(tk, fx.Float32(float(KV_DIM)), fx.Float32(1.0 / KV_DIM))
|
||||
+ fx.Float32(EPS)
|
||||
)
|
||||
# q: bf16 normed row, then vLLM's MXFP8 (a 32-group = 4 threads)
|
||||
ys = [bf16_round((xq[j] * rq) * qw[j]) for j in range(8)]
|
||||
amax = abs(ys[0])
|
||||
for y in ys[1:]:
|
||||
amax = fx.max(amax, abs(y))
|
||||
code = mx_code(butterfly(amax, (1, 2), fx.max))
|
||||
mul = mx_mul(code)
|
||||
if const_expr(c.get("index", False)): # noqa: SIM102
|
||||
# the bf16 qr, vLLM's indexer's input
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
words = [
|
||||
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
for i in range(4)
|
||||
]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(a["qr_out"]),
|
||||
t * (Q_RANK // 2) + 4 * tt,
|
||||
)
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([w0, w1], fx.Int32),
|
||||
rsrc(c["qx8"]),
|
||||
t * QX_WORDS + 2 * tt,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
if tt % 4 == 0:
|
||||
bo.buffer_store(
|
||||
code,
|
||||
rsrc(c["qx8s"]),
|
||||
t * (Q_RANK // 32) + tt // 4,
|
||||
cache_modifier=CM_DEV,
|
||||
)
|
||||
# kv: this thread's pair (2 tt, 2 tt + 1)
|
||||
kn = [bf16_round((xk[j] * rk) * kw[j]) for j in range(2)]
|
||||
e0 = bf16_round(kn[0] * cs - kn[1] * sn)
|
||||
e1 = bf16_round(kn[0] * sn + kn[1] * cs)
|
||||
# record quant of the nope dims: a 64-block = 32 threads (half a wave)
|
||||
amax = butterfly(fx.max(abs(e0), abs(e1)), (1, 2, 4, 8, 16), fx.max)
|
||||
rcode = ceil_exp(
|
||||
div_rn(
|
||||
fx.max(amax, fx.Float32(1e-4)), fx.Float32(448.0), fx.Float32(1.0 / 448.0)
|
||||
)
|
||||
)
|
||||
inv = mx_mul(rcode)
|
||||
q0 = clamp_fp8(e0 * inv)
|
||||
q1 = clamp_fp8(e1 * inv)
|
||||
n0 = xshfl(q0, 1)
|
||||
n1 = xshfl(q1, 1)
|
||||
word = fp8_pack4(q0, q1, n0, n1)
|
||||
rope_word = (
|
||||
fx.Vector.from_elements([e0, e1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
blk = a["swa_block"]
|
||||
sblock = slot // blk
|
||||
soff = slot % blk
|
||||
base = a["swa"] + fx.Int64(sblock) * fx.Int64(a["swa_stride"])
|
||||
rec = base + fx.Int64(soff * DATA)
|
||||
if mine & (tt % 32 == 0) & (tt < NOPE // 2):
|
||||
fx.ptr_store(rcode.bitcast(fx.Float32), red + 8 + tt // 32)
|
||||
gpu.barrier()
|
||||
if mine & (slot >= 0):
|
||||
if tt < NOPE // 2:
|
||||
if tt % 2 == 0:
|
||||
gstore(rec + fx.Int64(2 * tt), word, words=1)
|
||||
else:
|
||||
gstore(rec + fx.Int64(NOPE + 4 * (tt - NOPE // 2)), rope_word, words=1)
|
||||
if tt == 0:
|
||||
sb = [fx.ptr_load(red + 8 + b).bitcast(fx.Int32) for b in range(7)]
|
||||
w0 = sb[0] | (sb[1] << 8) | (sb[2] << 16) | (sb[3] << 24)
|
||||
w1 = sb[4] | (sb[5] << 8) | (sb[6] << 16)
|
||||
sc = base + fx.Int64(blk * DATA + soff * 8)
|
||||
gstore(sc, fx.Vector.from_elements([w0, w1], fx.Int32), words=2)
|
||||
publish(c, c["qrdy"], t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wq_b
|
||||
|
||||
|
||||
def wqb_rope(c, task0, pos_t, n):
|
||||
"""(cos, sin) of this thread's pair in ``stage_wqb``'s epilogue over a tile
|
||||
of n rows (``pos_t``: its token's position; every task of the CTA shares
|
||||
the pair's dims: tasks BLOCKS apart are whole heads apart)."""
|
||||
a = c["args"]
|
||||
rp = c["tid"] % (ROWS // 2)
|
||||
dim = (task0 * ROWS) % HEAD_DIM + 2 * rp
|
||||
jj = fx.max((dim - NOPE) >> 1, 0)
|
||||
return ld_f32(a["cos_sin"], pos_t * ROPE + jj), ld_f32(
|
||||
a["cos_sin"], pos_t * ROPE + HALF + jj
|
||||
)
|
||||
|
||||
|
||||
def wqb_pos(c, t0, n):
|
||||
"""This thread's token position for ``stage_wqb``'s epilogue of tile
|
||||
t0 .. t0 + n."""
|
||||
per = ROWS * n // 2
|
||||
tl = fx.min((c["tid"] % per) // (ROWS // 2), n - 1)
|
||||
return ld_i32(c["args"]["pos"], 2 * (t0 + tl))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_wqb(c, task0, bops, t0, n, rope):
|
||||
"""Rows 16 (task0 + BLOCKS k) .. of the rank's H x 512 query rows, every
|
||||
task k of the CTA: GEMVs -> bf16 -> GPT-J RoPE of a head's dims 448..
|
||||
(fp32) -> bf16 -> q."""
|
||||
tid, red, d = c["tid"], c["red"], c["d"]
|
||||
a = c["args"]
|
||||
nt = len(bops)
|
||||
for k in range_constexpr(nt):
|
||||
gemv_mfmas(c, Q_RANK, c["qxl"], c["qxsl"], red + k * RED, bops[k], rows=n)
|
||||
gpu.barrier()
|
||||
per = ROWS * n // 2 # bf16 pairs a task
|
||||
if tid < nt * per:
|
||||
k = tid // per
|
||||
u = tid % per
|
||||
rp = u % (ROWS // 2)
|
||||
tl = u // (ROWS // 2)
|
||||
t = plus(t0, tl)
|
||||
r0 = (task0 + BLOCKS * k) * ROWS + 2 * rp
|
||||
rk = red + k * RED
|
||||
v0 = bf16_round(row_sum(rk, 2 * rp, tl))
|
||||
v1 = bf16_round(row_sum(rk, 2 * rp + 1, tl))
|
||||
is_rope = (task0 * ROWS) % HEAD_DIM + 2 * rp >= NOPE
|
||||
cs, sn = rope
|
||||
e0 = is_rope.select(v0 * cs - v1 * sn, v0)
|
||||
e1 = is_rope.select(v0 * sn + v1 * cs, v1)
|
||||
word = (
|
||||
fx.Vector.from_elements([e0, e1], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
)
|
||||
bo.buffer_store(word, rsrc(a["q"]), (t * d.q_rows + r0) // 2)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kernel
|
||||
|
||||
|
||||
def front_lds(s: int, d: Dims):
|
||||
"""K1's LDS (the attention front's stages)."""
|
||||
rows = tile_rows(s)
|
||||
wqb_per_cta = d.wqb_tasks // BLOCKS
|
||||
|
||||
@fx.struct
|
||||
class FrontLds:
|
||||
red: fx.Array[fx.Float32, wqb_per_cta * RED + 16, 16]
|
||||
qxl: fx.Array[fx.Int32, rows * lds_row(QX_WORDS), 16]
|
||||
qxsl: fx.Array[fx.Int32, rows * lds_row(Q_RANK // 32), 16]
|
||||
xl: fx.Array[fx.Int32, rows * lds_row(KWORDS), 16]
|
||||
xsl: fx.Array[fx.Int32, rows * lds_row(KGROUPS), 16]
|
||||
|
||||
return FrontLds
|
||||
|
||||
|
||||
def front_context(key, lds, args: dict, scratch, tag, layout=None) -> dict:
|
||||
"""The front stages' context: LDS views, mailbox (tag ``tag``), scratch
|
||||
regions, arguments (``args``: the kernel's, by name)."""
|
||||
s, d = key.tokens, Dims(key.tp)
|
||||
tid = fx.thread_idx.x
|
||||
c = {
|
||||
"S": s,
|
||||
"tid": tid,
|
||||
"bid": fx.block_idx.x,
|
||||
"lane": tid % 64,
|
||||
"wave": tid // 64,
|
||||
"red": lds.red.ptr,
|
||||
"xl": lds.xl.ptr,
|
||||
"xsl": lds.xsl.ptr,
|
||||
"qxl": lds.qxl.ptr,
|
||||
"qxsl": lds.qxsl.ptr,
|
||||
"mb": Mailbox(tag),
|
||||
"d": d,
|
||||
"ratio": key.ratio,
|
||||
"ldbf": lambda ptr, i: fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
),
|
||||
"args": args,
|
||||
}
|
||||
for region, (off, _) in (layout or front_scratch(s, d)).items():
|
||||
c[region] = scratch + fx.Int64(off)
|
||||
return c
|
||||
|
||||
|
||||
@traced
|
||||
def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
|
||||
"""The front's stages on this CTA. ``x8_given``: a stage before this one
|
||||
(the mono layer's seam norm) publishes X8 / XRDY for every token (no input
|
||||
quantization here). ``before_wqkv()`` runs ahead of the wqkv weights' loads
|
||||
(a latency-critical stage of the caller's), ``after_wqkv()`` right after."""
|
||||
s = key.tokens
|
||||
wqb_tasks = Dims(key.tp).wqb_tasks
|
||||
a = c["args"]
|
||||
tid = c["tid"]
|
||||
tiles = token_tiles(s)
|
||||
wqb_per_cta = wqb_tasks // BLOCKS
|
||||
assert wqb_tasks % BLOCKS == 0
|
||||
n_wqkv = WQKV_TASKS * WQKV_KPARTS # 224
|
||||
assert n_wqkv <= BLOCKS
|
||||
n_free = BLOCKS - n_wqkv # CTAs without a wqkv part: xq, kt
|
||||
QKV0 = 0 # on wqkv CTAs
|
||||
WQB0 = 0
|
||||
in_cta_quant = s <= TILE and not x8_given
|
||||
xq_stage = s > TILE and not x8_given
|
||||
xq_units = WQKV_KPARTS * s
|
||||
xq_per = (xq_units + n_free - 1) // n_free
|
||||
stamp(c, 0)
|
||||
|
||||
# ---- Loads complete in issue order, so every CTA issues its latency-
|
||||
# critical loads before its later stages' weights: a wqkv CTA its first
|
||||
# input tile and its wqkv part, the next tiles while it quantizes, then
|
||||
# its wq_b tasks; a CTA with a key-table task those after that work.
|
||||
# The q / kv norms run on wqkv CTAs, their partials' wait behind the
|
||||
# wq_b weights that are due by then anyway.
|
||||
wqkv_unit = first_task(bid, 0)
|
||||
has_wqkv = wqkv_unit < n_wqkv
|
||||
task = fx.min(wqkv_unit, n_wqkv - 1) % WQKV_TASKS
|
||||
part = fx.min(wqkv_unit, n_wqkv - 1) // WQKV_TASKS
|
||||
wqb_task = first_task(bid, WQB0)
|
||||
free = bid - n_wqkv
|
||||
is_free = free >= 0
|
||||
if const_expr(in_cta_quant):
|
||||
xregs = [x_half_loads(c, part, has_wqkv, *tiles[0])]
|
||||
if const_expr(xq_stage):
|
||||
xunits = [free + n_free * i for i in range(xq_per)]
|
||||
xlive = [is_free & (u < xq_units) for u in xunits]
|
||||
xq_regs = [
|
||||
xq_loads(c, fx.min(fx.max(u, 0), xq_units - 1), lv)
|
||||
for u, lv in zip(xunits, xlive)
|
||||
]
|
||||
if const_expr(before_wqkv is not None):
|
||||
before_wqkv()
|
||||
kops = gemv_loads(
|
||||
c,
|
||||
a["wqkv"],
|
||||
a["wqkv_s"],
|
||||
HIDDEN,
|
||||
task * ROWS,
|
||||
kbeg=part * KSTEPS,
|
||||
nsteps=KSTEPS,
|
||||
pred=has_wqkv,
|
||||
)
|
||||
if const_expr(after_wqkv is not None):
|
||||
after_wqkv()
|
||||
if const_expr(xq_stage): # noqa: SIM102 (a build-time and a run-time test)
|
||||
if is_free:
|
||||
for i in range_constexpr(xq_per):
|
||||
stage_xq(
|
||||
c, fx.min(fx.max(xunits[i], 0), xq_units - 1), xq_regs[i], xlive[i]
|
||||
)
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if tid < xq_per:
|
||||
u = free + n_free * tid
|
||||
if u < xq_units:
|
||||
c["mb"].put(c["xrdy"], u, fx.Int32(1))
|
||||
if const_expr(not c.get("index", False)):
|
||||
# an index source's top-k comes after this launch: K2 builds its table
|
||||
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
|
||||
stage_kt(c, t)
|
||||
stamp(c, 1)
|
||||
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
|
||||
|
||||
# ---- wqkv_a: one (row tile, K half) a CTA, every token tile; the wq_b
|
||||
# weights issued behind the input rows' loads
|
||||
def wqb_weights():
|
||||
return [
|
||||
gemv_loads(c, a["wqb"], a["wqb_s"], Q_RANK, (wqb_task + BLOCKS * i) * ROWS)
|
||||
for i in range(wqb_per_cta)
|
||||
]
|
||||
|
||||
if const_expr(in_cta_quant):
|
||||
if has_wqkv:
|
||||
quant_x_half(c, xregs[0], tiles[0][1])
|
||||
bops = wqb_weights()
|
||||
if has_wqkv:
|
||||
stage_wqkv(c, task, part, kops, *tiles[0], has_wqkv)
|
||||
else:
|
||||
await_flags(c, c["xrdy"], s, lambda u: WQKV_KPARTS * u + part)
|
||||
xr = (
|
||||
c["x8"],
|
||||
c["x8s"],
|
||||
X_WORDS,
|
||||
part * KWORDS,
|
||||
KWORDS,
|
||||
X_GROUPS,
|
||||
part * KGROUPS,
|
||||
KGROUPS,
|
||||
)
|
||||
regs = rows_loads(c, *xr, tiles[0][0], tiles[0][1], pred=has_wqkv)
|
||||
if const_expr(len(tiles) == 1):
|
||||
bops = wqb_weights()
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, KWORDS, KGROUPS, c["xl"], c["xsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
t1, n1 = tiles[ti + 1]
|
||||
regs = rows_loads(c, *xr, t1, n1, pred=has_wqkv)
|
||||
if const_expr(ti + 2 == len(tiles)):
|
||||
bops = wqb_weights()
|
||||
stage_wqkv(c, task, part, kops, t0, n, has_wqkv)
|
||||
stamp(c, 2)
|
||||
|
||||
# ---- q / kv norms, KV insert
|
||||
for t in range(first_task(bid, QKV0), s, BLOCKS):
|
||||
stage_qkv(c, t)
|
||||
stamp(c, 3)
|
||||
|
||||
# ---- wq_b: every task's rows of a token tile at a time, the next
|
||||
# tile's rows in flight
|
||||
ropes = [wqb_rope(c, wqb_task, poss[ti], tiles[ti][1]) for ti in range(len(tiles))]
|
||||
await_flags(c, c["qrdy"], s, lambda u: u)
|
||||
qx = (c["qx8"], c["qx8s"], QX_WORDS, 0, QX_WORDS, Q_RANK // 32, 0, Q_RANK // 32)
|
||||
regs = rows_loads(c, *qx, *tiles[0])
|
||||
for ti in range_constexpr(len(tiles)):
|
||||
t0, n = tiles[ti]
|
||||
rows_store(c, regs, QX_WORDS, Q_RANK // 32, c["qxl"], c["qxsl"], n)
|
||||
if const_expr(ti + 1 < len(tiles)):
|
||||
regs = rows_loads(c, *qx, *tiles[ti + 1])
|
||||
stage_wqb(c, wqb_task, bops, t0, n, ropes[ti])
|
||||
stamp(c, 4)
|
||||
@@ -1,234 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 GEMV of the mega kernel: 16 rows of a row-major e4m3 weight [N, K]
|
||||
(vLLM's loaded layout, E8M0 scales in 32x32 blocks [N / 32, K / 32]) against
|
||||
every token of a tile, its MXFP8 rows in LDS, on the gfx950 scaled MFMA.
|
||||
|
||||
A task may cover a window of K (``kbeg`` .. ``kbeg + nsteps`` 128-steps): a
|
||||
split-K part, its activations in LDS holding just that window. The weight
|
||||
operands are loaded first (all in flight, before the task waits on its
|
||||
activations); wave w takes the window's steps w, w + 8, ... (or, ``split``, a
|
||||
contiguous range of waves 0 .. split - 1), its 16 x 16 partial in ``red``,
|
||||
summed in wave order by ``row_sum``. Adapted from ATOM's
|
||||
``attn_pre.gemv_fp8_*``, which reads aiter's (16, 16)-preshuffled layout.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from .device import THREADS, WAVES, mfma_scaled, rsrc, tile_map, traced
|
||||
|
||||
# a token's LDS row of MXFP8 words / E8M0 codes, padded by 16 B so 16 rows read
|
||||
# by one MFMA operand do not share a bank group
|
||||
LDS_PAD = 4
|
||||
RED = WAVES * 64 * 4 # one task's accumulators in ``red`` (f32)
|
||||
|
||||
|
||||
def lds_row(words):
|
||||
return words + LDS_PAD
|
||||
|
||||
|
||||
def lds_at(u, row):
|
||||
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
|
||||
return u + u // row * LDS_PAD
|
||||
|
||||
|
||||
def _steps(c, nsteps, split, tiled):
|
||||
"""This wave's steps of the window: [(local step, live)]."""
|
||||
wave = c["wave"]
|
||||
if const_expr(split is None):
|
||||
per = (nsteps + WAVES - 1) // WAVES
|
||||
ragged = nsteps % WAVES != 0
|
||||
return [
|
||||
(wave + WAVES * i, (wave + WAVES * i < nsteps) if ragged else None)
|
||||
for i in range(per)
|
||||
]
|
||||
assert split <= WAVES and nsteps % split == 0
|
||||
per = nsteps // split
|
||||
part = wave % split if tiled else fx.min(wave, split - 1)
|
||||
return [(part * per + i, None) for i in range(per)]
|
||||
|
||||
|
||||
def _step_load(c, r_w, r_ws, k, row0, st, pred=None):
|
||||
"""Global step ``st``'s weight operand of rows row0 .. +16: two 16 B chunks a
|
||||
lane (K bytes 16 j .. and 64 + 16 j .. of the step, row row0 + lane % 16)
|
||||
and the E8M0 byte of its K block. ``pred`` false: masked (out of range: no
|
||||
memory traffic, zeros)."""
|
||||
lane = c["lane"]
|
||||
j = lane // 16
|
||||
row = row0 + lane % 16
|
||||
base = row * k + 128 * st + 16 * j
|
||||
lo = fx.Vector(bo.buffer_load(r_w, base // 4, vec_width=4, dtype=T.i32))
|
||||
hi = fx.Vector(bo.buffer_load(r_w, (base + 64) // 4, vec_width=4, dtype=T.i32))
|
||||
kg = k // 32
|
||||
wsi = (row0 // 32) * kg + st * 4 + j
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def gemv_loads(
|
||||
c, w, ws, k, row0, split=None, tiled=False, kbeg=0, nsteps=None, pred=None
|
||||
):
|
||||
"""The weight operands of rows row0 .. +16 (``tiled``: row0 + 16 (wave //
|
||||
split)) over steps kbeg .. kbeg + nsteps for this wave, all in flight
|
||||
(``pred`` false: masked off, no memory traffic)."""
|
||||
nsteps = k // 128 if nsteps is None else nsteps
|
||||
if const_expr(pred is None):
|
||||
r_w, r_ws = rsrc(w), rsrc(ws)
|
||||
else:
|
||||
# a masked-off CTA reads through a zero-size buffer: zeros, no traffic
|
||||
nb = pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
|
||||
r_w, r_ws = rsrc(w, nb), rsrc(ws, nb)
|
||||
if const_expr(tiled):
|
||||
row0 = row0 + 16 * (c["wave"] // split)
|
||||
return [
|
||||
_step_load(
|
||||
c,
|
||||
r_w,
|
||||
r_ws,
|
||||
k,
|
||||
row0,
|
||||
kbeg + (st if live is None else fx.min(st, nsteps - 1)),
|
||||
pred,
|
||||
)
|
||||
for st, live in _steps(c, nsteps, split, tiled)
|
||||
]
|
||||
|
||||
|
||||
def _step_mfma(c, nsteps, xl, xsl, st, ops, acc, live, rows):
|
||||
lane = c["lane"]
|
||||
lo, hi, sa = ops
|
||||
x_row, s_row = lds_row(nsteps * 32), lds_row(nsteps * 4)
|
||||
j = lane // 16
|
||||
col = fx.min(lane % 16, rows - 1)
|
||||
if const_expr(live is not None):
|
||||
st = fx.min(st, nsteps - 1)
|
||||
av = fx.Vector.from_elements(
|
||||
[lo[d] for d in range(4)] + [hi[d] for d in range(4)], fx.Int32
|
||||
)
|
||||
xlo = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xhi = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + 16 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xv = fx.Vector.from_elements(
|
||||
[xlo[d] for d in range(4)] + [xhi[d] for d in range(4)], fx.Int32
|
||||
)
|
||||
sb = fx.ptr_load(xsl + (col * s_row + st * 4 + j))
|
||||
if const_expr(live is not None):
|
||||
# a step past the end adds zero at unit scale
|
||||
av = fx.Vector.from_elements(
|
||||
[live.select(av[d], fx.Int32(0)) for d in range(8)], fx.Int32
|
||||
)
|
||||
sa = live.select(sa, fx.Int32(127))
|
||||
sb = live.select(sb, fx.Int32(127))
|
||||
return mfma_scaled(av, xv, acc, sa, sb)
|
||||
|
||||
|
||||
def gemv_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=16, nsteps=None):
|
||||
"""``gemv_loads``' operands against ``rows`` tokens' MXFP8 rows of the
|
||||
window in LDS (``xl`` words, ``xsl`` codes, rows ``lds_row`` of the window)
|
||||
-> ``red[wave][lane]`` (a 16 x 16 C: row 4 (lane // 16) + i, token lane % 16)."""
|
||||
lane, wave = c["lane"], c["wave"]
|
||||
nsteps = k // 128 if nsteps is None else nsteps
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i, (st, live) in enumerate(_steps(c, nsteps, split, tiled)):
|
||||
acc = _step_mfma(c, nsteps, xl, xsl, st, ops[i], acc, live, rows)
|
||||
if const_expr(split is not None and not tiled):
|
||||
acc = fx.Vector.from_elements(
|
||||
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
|
||||
fx.Float32,
|
||||
)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
|
||||
|
||||
def row_sum(red, r, t, waves=None):
|
||||
"""Row r, token t of the waves' accumulators, summed in wave order."""
|
||||
tot = fx.Float32(0.0)
|
||||
for w in waves if waves is not None else range(WAVES):
|
||||
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
|
||||
return tot
|
||||
|
||||
|
||||
def span_items(width, n):
|
||||
"""A [n, width] span of 16 B items: rows a round of THREADS threads, rounds."""
|
||||
wq = width // 4
|
||||
rpr = THREADS // wq
|
||||
assert rpr >= 1, width
|
||||
return wq, rpr, (n + rpr - 1) // rpr
|
||||
|
||||
|
||||
def rows_loads(
|
||||
c,
|
||||
words_base,
|
||||
codes_base,
|
||||
stride_w,
|
||||
off_w,
|
||||
words,
|
||||
stride_g,
|
||||
off_g,
|
||||
groups,
|
||||
t0,
|
||||
n,
|
||||
pred=None,
|
||||
):
|
||||
"""Tokens t0 .. t0 + n's published MXFP8 row slices -- ``words`` i32 from
|
||||
``off_w`` and ``groups`` codes (an i32 each) from ``off_g``, rows
|
||||
``stride_w`` / ``stride_g`` i32 apart -- loaded (issued, not waited on),
|
||||
16 B a lane-item: a thread one column, rows a round apart. Plain loads:
|
||||
their flags are seen, and nothing in the launch read these lines before.
|
||||
``pred`` false: masked off (zeros, no memory traffic)."""
|
||||
tid = c["tid"]
|
||||
nb = None if pred is None else pred.select(fx.Int64(0xFFFFFFFF), fx.Int64(0))
|
||||
out = []
|
||||
for base, stride, off, width in (
|
||||
(words_base, stride_w, off_w, words),
|
||||
(codes_base, stride_g, off_g, groups),
|
||||
):
|
||||
wq, rpr, rounds = span_items(width, n)
|
||||
row0, col = tile_map(tid, wq)
|
||||
a0 = col * 4 + off + t0 * stride
|
||||
r = rsrc(base, nb)
|
||||
regs = []
|
||||
for i in range(rounds):
|
||||
row = row0 + rpr * i
|
||||
if (i + 1) * rpr > n or rpr * wq < THREADS:
|
||||
row = fx.min(row, n - 1)
|
||||
regs.append(
|
||||
fx.Vector(
|
||||
bo.buffer_load(r, row * stride + a0, vec_width=4, dtype=T.i32)
|
||||
)
|
||||
)
|
||||
out.append(regs)
|
||||
return out
|
||||
|
||||
|
||||
@traced
|
||||
def _store_span(c, regs, width, dst, n):
|
||||
tid = c["tid"]
|
||||
wq, rpr, _ = span_items(width, n)
|
||||
row0, col = tile_map(tid, wq)
|
||||
lrow = lds_row(width)
|
||||
base = dst + (row0 * lrow + col * 4)
|
||||
for i in range_constexpr(len(regs)):
|
||||
if const_expr((i + 1) * rpr <= n and rpr * wq == THREADS):
|
||||
fx.ptr_store(regs[i], base + i * rpr * lrow)
|
||||
else:
|
||||
if (row0 + rpr * i < n) & (tid < rpr * wq):
|
||||
fx.ptr_store(regs[i], base + i * rpr * lrow)
|
||||
|
||||
|
||||
def rows_store(c, regs, words, groups, xl, xsl, n):
|
||||
"""``rows_loads``' registers into LDS rows 0 .. n (``lds_row`` strides)."""
|
||||
_store_span(c, regs[0], words, xl, n)
|
||||
_store_span(c, regs[1], groups, xsl, n)
|
||||
gpu.barrier()
|
||||
@@ -1,173 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Geometry of the DeepSeek-V4.1 attention mega kernel: model widths, a TP
|
||||
rank's share of them, task counts and the scratch (mailbox) layout."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
HIDDEN = 5120
|
||||
Q_RANK = 1280
|
||||
KV_DIM = 512
|
||||
QKV_ROWS = Q_RANK + KV_DIM # 1792
|
||||
HEAD_DIM = 512
|
||||
NOPE = 448
|
||||
ROPE = 64
|
||||
HALF = ROPE // 2
|
||||
N_HEADS = 64
|
||||
O_GROUPS = 8
|
||||
O_RANK = 1024
|
||||
WINDOW = 128
|
||||
TOPK = 512
|
||||
EPS = 1e-20
|
||||
|
||||
ROWS = 16 # GEMV rows a task
|
||||
TILE = 16 # tokens a GEMV pass (an MFMA's N)
|
||||
HEAD_TILE = 16 # heads an attention MFMA takes
|
||||
WOA_ROWS = 32 # a wo_a task's rows: one MX group of its output
|
||||
MAX_TOKENS = 48
|
||||
|
||||
# KV record (V4 fp8_ds_mla): 576 B data rows (448 e4m3 | 64 bf16), then 8 B of
|
||||
# scales a token (7 UE8M0 + pad) after the block's rows
|
||||
DATA = 576
|
||||
SCALE_BYTES = 8
|
||||
|
||||
# attention: a split is 16 keys; a token has at most TOPK + WINDOW keys
|
||||
BK = 16
|
||||
SPLITS = 64 # split slots a (token, head tile): 1024 keys >= TOPK + WINDOW
|
||||
COLS = 32 # PV columns a group (one MX group of the output)
|
||||
GROUPS = HEAD_DIM // COLS # 16
|
||||
|
||||
PAIR = 8 # a mailbox (value, tag) pair
|
||||
|
||||
|
||||
def cdiv(a, b):
|
||||
return -(-a // b)
|
||||
|
||||
|
||||
def round_up(a, b):
|
||||
return cdiv(a, b) * b
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dims:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.tp in (2, 4), self.tp
|
||||
|
||||
@property
|
||||
def heads(self) -> int:
|
||||
return N_HEADS // self.tp
|
||||
|
||||
@property
|
||||
def head_tiles(self) -> int:
|
||||
return self.heads // HEAD_TILE
|
||||
|
||||
@property
|
||||
def groups(self) -> int:
|
||||
return O_GROUPS // self.tp
|
||||
|
||||
@property
|
||||
def group_k(self) -> int:
|
||||
"""wo_a's K a group: its eight heads' outputs."""
|
||||
return self.heads // self.groups * HEAD_DIM # 4096
|
||||
|
||||
@property
|
||||
def o_rows(self) -> int:
|
||||
return self.groups * O_RANK
|
||||
|
||||
@property
|
||||
def q_rows(self) -> int:
|
||||
return self.heads * HEAD_DIM
|
||||
|
||||
@property
|
||||
def wqb_tasks(self) -> int:
|
||||
return self.q_rows // ROWS
|
||||
|
||||
@property
|
||||
def woa_tasks(self) -> int:
|
||||
return self.o_rows // WOA_ROWS
|
||||
|
||||
@property
|
||||
def wob_tasks(self) -> int:
|
||||
return HIDDEN // ROWS # 320
|
||||
|
||||
|
||||
WQKV_TASKS = QKV_ROWS // ROWS # 112
|
||||
WQKV_KPARTS = 2 # wqkv's K split: 224 (row tile, K half) parts
|
||||
KEYS = TOPK + WINDOW # a token's attention keys at most
|
||||
|
||||
|
||||
def token_tiles(s):
|
||||
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
|
||||
|
||||
|
||||
def tile_rows(s):
|
||||
return min(s, TILE)
|
||||
|
||||
|
||||
def pair_layout(items, start=0, align=256):
|
||||
out, off = {}, start
|
||||
for name, pairs in items:
|
||||
off = round_up(off, align)
|
||||
out[name] = (off, pairs * PAIR)
|
||||
off += pairs * PAIR
|
||||
return out
|
||||
|
||||
|
||||
def byte_layout(items, start=0, align=256):
|
||||
out, off = {}, start
|
||||
for name, nbytes in items:
|
||||
off = round_up(off, align)
|
||||
out[name] = (off, nbytes)
|
||||
off += nbytes
|
||||
return out
|
||||
|
||||
|
||||
def front_scratch(s: int, d: Dims) -> dict:
|
||||
"""K1's scratch regions -> (byte offset, bytes): published data (plain
|
||||
words behind a flag) and mailbox pairs."""
|
||||
return byte_layout(
|
||||
[
|
||||
("x8", s * HIDDEN), # e4m3 words (S > 16: quantized once, shared)
|
||||
("x8s", s * HIDDEN // 32 * 4), # an i32 code a group
|
||||
("xrdy", WQKV_KPARTS * s * PAIR), # a flag a (token, K half)
|
||||
("qkvp", WQKV_KPARTS * s * QKV_ROWS * PAIR),
|
||||
("qx8", s * Q_RANK), # e4m3 words
|
||||
("qx8s", s * Q_RANK // 32 * 4), # an i32 code a group
|
||||
("qrdy", s * PAIR),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def back_scratch(s: int, d: Dims, start: int = 0) -> dict:
|
||||
"""K2's scratch regions, laid out past ``start``: published data (plain)
|
||||
and flag pairs."""
|
||||
ck = (
|
||||
64 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= 256 else 128
|
||||
) # back.chunk_keys
|
||||
nchunk = cdiv(KEYS, ck)
|
||||
return byte_layout(
|
||||
[
|
||||
("part", s * nchunk * d.heads * HEAD_DIM * 2), # bf16 partials
|
||||
("pstat", s * nchunk * d.heads * 8), # (max, sum) f32
|
||||
("srdy", s * nchunk * PAIR),
|
||||
("xo", s * d.q_rows), # e4m3 words
|
||||
("xos", s * d.q_rows // 32 * 4), # an i32 code a group
|
||||
("crdy", s * d.heads // 8 * PAIR),
|
||||
("x8b", s * d.o_rows),
|
||||
("x8bs", s * d.o_rows // 32 * 4),
|
||||
("ardy", d.woa_tasks * PAIR),
|
||||
],
|
||||
start=start,
|
||||
)
|
||||
|
||||
|
||||
def front_end(s: int, d: Dims) -> int:
|
||||
"""Where K2's regions start: past K1's, which share the launch tag."""
|
||||
return round_up(max(o + n for o, n in front_scratch(s, d).values()), 256)
|
||||
|
||||
|
||||
def scratch_bytes(s: int, d: Dims) -> int:
|
||||
b = back_scratch(s, d, start=front_end(s, d))
|
||||
return round_up(max(o + n for o, n in b.values()), 4096)
|
||||
@@ -1,9 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/__init__.py
|
||||
"""The mono kernels' shared mechanisms: ``ops`` (wave / math primitives, the TP
|
||||
group's peer buffers), ``sync`` (the tagged mailbox), ``mx`` (FP8 / MX
|
||||
arithmetic), ``plan`` (execution model, layouts, build keys) and
|
||||
``peer_memory`` (symmetric peer buffers)."""
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/mx.py
|
||||
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
|
||||
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
|
||||
tile's operand loads and the scaled MFMA that takes them.
|
||||
|
||||
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
|
||||
|
||||
FP8_MAX = 448.0
|
||||
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
|
||||
FP8, FP4 = 0, 4
|
||||
UNIT_SCALE = 127 # the E8M0 code of 1.0
|
||||
|
||||
|
||||
def clamp_fp8(v):
|
||||
"""``v`` clamped to the E4M3 range."""
|
||||
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
||||
|
||||
|
||||
def code_ceil(v):
|
||||
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
|
||||
clamped to [1, 254]: the codes a scale may take."""
|
||||
bits = v.bitcast(fx.Int32)
|
||||
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
||||
fx.Int32(1), fx.Int32(0)
|
||||
)
|
||||
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
|
||||
|
||||
|
||||
def rcp_pow2(code):
|
||||
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
|
||||
bit for bit, a multiply where the division is a dozen instructions."""
|
||||
return ((254 - code) << 23).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def scale_index(row, col, cols):
|
||||
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
|
||||
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
|
||||
r32, a, b = row // 32, (row // 16) % 2, row % 16
|
||||
c8, d, e = col // 8, (col // 4) % 2, col % 4
|
||||
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
|
||||
|
||||
|
||||
def fp4_tile_load(r_w, base, rg, k, st, at):
|
||||
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
|
||||
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
|
||||
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
|
||||
Nontemporal: a weight a step reads once."""
|
||||
return fx.Vector(
|
||||
bo.buffer_load(
|
||||
r_w,
|
||||
base + rg * (2 * k) + st * 256 + at,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=CM_NT,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
|
||||
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
|
||||
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
|
||||
byte ``sb_byte``)."""
|
||||
return fx.Vector(
|
||||
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
||||
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
|
||||
)
|
||||
)
|
||||
@@ -1,300 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/ops.py
|
||||
# atom/mono/device/ranks.py
|
||||
# atom/mono/device/stamps.py
|
||||
"""Device-side primitives of the mono kernels: wave reductions and shuffles, the
|
||||
approximate hardware math, bf16 / fp8 packing, buffer resources, the
|
||||
execution-model constants and the kernel symbol naming.
|
||||
|
||||
Ported from the FlyDSL GLM-5 layer kernel (ROCm/FlyDSL branch
|
||||
``glm5-mla-moe-layer-monokernel`` @ 1e28fd8a,
|
||||
``kernels/mla_moe_layer/shared_reuse_moe_kernel.py``). Only published flydsl APIs
|
||||
and aiter's vendored ``buffer_ops`` / ``dpp_utils`` are used.
|
||||
|
||||
Every function but ``kernel_symbol`` is traced inside a ``@flyc.kernel`` body.
|
||||
|
||||
Raw boundaries kept, for want of an equivalent on the pinned FlyDSL:
|
||||
``buffer_ops`` loads / stores (the kernels take raw Int64 addresses, no tensors
|
||||
for ``make_buffer_tensor``); raw MFMA intrinsics (hand-laid operands: permlane
|
||||
transposes, fp8 packing, MXFP4 scales); the LLVM intrinsics here (``expect``,
|
||||
``s_memrealtime``, the approximate rsq / rcp / exp2) and the struct unpacking of
|
||||
``permlane*_swap``.
|
||||
|
||||
Also the TP group seen from inside a kernel (every rank's peer buffer, an
|
||||
all-reduce's rank-ordered sum) and a timeline build's phase stamps."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
|
||||
from flydsl._mlir.dialects import llvm
|
||||
from flydsl.compiler.ast_rewriter import ASTRewriter
|
||||
from flydsl.expr import const_expr, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T, as_ir_value
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES
|
||||
|
||||
# ---------------------------------------------------------------- primitives
|
||||
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16. SC1 alone is device
|
||||
# scope (past the per-XCD caches); SC0 | SC1 is system scope (peers over XGMI).
|
||||
CM_DEV = 16
|
||||
|
||||
|
||||
CM_SYS = 17
|
||||
|
||||
|
||||
# NT = 2: nontemporal. A weight a layer reads once streams past the caches it
|
||||
# would only evict the step's reused data from (the MoE at 48 rows: 143 -> 133 us)
|
||||
CM_NT = 2
|
||||
|
||||
|
||||
def rsrc(addr, nbytes=None):
|
||||
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
|
||||
|
||||
|
||||
def permlane_swap(off, x, y):
|
||||
"""v_permlane{32,16}_swap on registers (x, y): off 32 trades x's upper-half
|
||||
lanes with y's lower-half lanes; off 16 trades x's odd 16-lane rows with y's
|
||||
even rows (each within its half). -> (x', y')."""
|
||||
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
|
||||
pr = swap(
|
||||
llvm.StructType.get_literal([T.i32, T.i32]),
|
||||
as_ir_value(fx.Int32(x)),
|
||||
as_ir_value(fx.Int32(y)),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
|
||||
|
||||
|
||||
def lane_gather(v, src_lane):
|
||||
"""Lane ``src_lane``'s value of v, a source lane a lane (ds_bpermute)."""
|
||||
return fx.Int32(
|
||||
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
|
||||
)
|
||||
|
||||
|
||||
def memrealtime():
|
||||
"""s_memrealtime: the 100 MHz device clock (no FlyDSL wrapper)."""
|
||||
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
|
||||
|
||||
|
||||
def fresh(v):
|
||||
"""``v`` (a per-lane Int32) through an opaque move, here: the lane math built
|
||||
on it is computed at its use, where LLVM would otherwise hoist it out of a task
|
||||
loop or CSE it across stages into registers held for the whole kernel (spills,
|
||||
and a kernel with scratch is refused: ``runtime.widths``)."""
|
||||
return fx.Int32(
|
||||
llvm.inline_asm(
|
||||
T.i32,
|
||||
[fx.Int32(v).ir_value()],
|
||||
"v_mov_b32 $0, $1",
|
||||
"=v,v",
|
||||
has_side_effects=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def uniform(v):
|
||||
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
|
||||
|
||||
|
||||
def _hw_f32(name, x):
|
||||
return fx.Float32(
|
||||
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
|
||||
)
|
||||
|
||||
|
||||
def hw_rsq(x):
|
||||
"""v_rsq_f32 (~1 ulp, no libm range fixups)."""
|
||||
return _hw_f32("llvm.amdgcn.rsq.f32", x)
|
||||
|
||||
|
||||
def hw_rcp(x):
|
||||
return _hw_f32("llvm.amdgcn.rcp.f32", x)
|
||||
|
||||
|
||||
def hw_exp2(x):
|
||||
return _hw_f32("llvm.amdgcn.exp2.f32", x)
|
||||
|
||||
|
||||
def div_rn(x, d, r):
|
||||
"""``x / d`` rounded to nearest, bit-identical to the IEEE division, given
|
||||
``r = 1.0 / d`` from one true division per divisor (Markstein: a quotient
|
||||
within 1 ulp plus its exact fma residual times RN(1/d) rounds correctly)."""
|
||||
q = x * r
|
||||
return fx.Float32(fmath.fma(fx.Float32(fmath.fma(-q, d, x)), r, q))
|
||||
|
||||
|
||||
def xshfl(v, off):
|
||||
"""Value of lane ``lane ^ off``: offsets 32 / 16 via v_permlane*_swap, in-row
|
||||
offsets via DPP (VALU latency, not LDS). Powers of two only: any other
|
||||
offset would take the lane ^ 1 path."""
|
||||
assert off in (1, 2, 4, 8, 16, 32), off
|
||||
if off >= 16:
|
||||
return v.shuffle_xor(off, 64)
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
if off == 8:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
|
||||
elif off == 4:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
|
||||
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
|
||||
elif off == 2:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
|
||||
else:
|
||||
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
|
||||
return y.bitcast(fx.Float32) if is_f else y
|
||||
|
||||
|
||||
def xred(v, off, op):
|
||||
"""op(v, lane ^ off) for a symmetric op; 32 / 16 take both halves of one swap."""
|
||||
if off < 16:
|
||||
return op(v, xshfl(v, off))
|
||||
is_f = isinstance(v, fx.Float32)
|
||||
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
||||
a, b = permlane_swap(off, x, x)
|
||||
if is_f:
|
||||
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
|
||||
return op(type(v)(a), type(v)(b))
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return a + b
|
||||
|
||||
|
||||
def butterfly(v, offsets, op=_add):
|
||||
"""Butterfly reduction over the given xor offsets, in the given order.
|
||||
|
||||
The order is the numerical contract: the aiter per-head norm reduces 32
|
||||
lanes with offsets 16, 8, 4, 2, 1, and matching it keeps rstd bit-exact."""
|
||||
for off in offsets:
|
||||
v = xred(v, off, op)
|
||||
return v
|
||||
|
||||
|
||||
def traced(fn):
|
||||
"""Apply flydsl's AST rewriting to a helper, so its ``while`` / ``if`` over
|
||||
traced values lower to scf ops. Kernels rewrite only their own body."""
|
||||
return ASTRewriter.transform(fn)
|
||||
|
||||
|
||||
LOAD_BATCH = 4 # rounds of a strided loop whose loads go out together
|
||||
|
||||
|
||||
@traced
|
||||
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
|
||||
"""A CTA's strided loop over elements v = tid + THREADS i < n (n a Python
|
||||
int): ``load(v)`` of ``batch`` rounds issued before any ``use(v, loaded)``.
|
||||
A round whose ``use`` stores to global memory keeps the next round's loads
|
||||
behind it (they may alias), so the rounds' loads would go out in series. A
|
||||
round past n loads element n - 1 (in range) and uses nothing."""
|
||||
rounds = -(-n // THREADS)
|
||||
for b0 in range_constexpr(0, rounds, batch):
|
||||
idx = range(b0, min(b0 + batch, rounds))
|
||||
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
|
||||
for i, g in zip(idx, got):
|
||||
v = tid + THREADS * i
|
||||
if v < n:
|
||||
use(v, g)
|
||||
|
||||
|
||||
def mfma_bf16(a, b, c):
|
||||
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
|
||||
|
||||
|
||||
def row_sum(red, r, t, waves=None):
|
||||
"""Row r, token t of the waves' 16x16 MFMA accumulators (wave w's at ``red``
|
||||
+ (w * 64 + lane) * 4), summed in wave order from 0 (``waves``: which, in
|
||||
order; all of them by default)."""
|
||||
tot = fx.Float32(0.0)
|
||||
for w in waves if waves is not None else range(WAVES):
|
||||
tot = tot + fx.ptr_load(red + ((w * 64 + 16 * (r // 4) + t) * 4 + r % 4))
|
||||
return tot
|
||||
|
||||
|
||||
def ballot(pred):
|
||||
"""The wave's lanes where ``pred`` holds, a bit a lane."""
|
||||
return fx.Int64(rocdl.ballot(T.i64, as_ir_value(pred)))
|
||||
|
||||
|
||||
def popcount(x):
|
||||
return fx.Int32(fx.ctpop(x))
|
||||
|
||||
|
||||
def lanes_below(bal, lane):
|
||||
"""How many of the lanes in ballot ``bal`` are below ``lane``."""
|
||||
return popcount(bal & ((fx.Int64(1) << fx.Int64(lane)) - 1))
|
||||
|
||||
|
||||
def load_ptr64(table, i):
|
||||
"""Entry ``i`` of an i64 table in memory, wave-uniform (each half through
|
||||
``uniform``): a peer's buffer base, a kernel argument block's pointer."""
|
||||
pw = fx.Vector(bo.buffer_load(rsrc(table), i * 2, vec_width=2, dtype=T.i32))
|
||||
return (fx.Int64(uniform(pw[1])) << 32) | fx.Int64(fx.Uint32(uniform(pw[0])))
|
||||
|
||||
|
||||
def bf16_round(a):
|
||||
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
|
||||
|
||||
|
||||
def bf16_pair(a, b):
|
||||
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
|
||||
return (
|
||||
fx.Vector.from_elements([a, b], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Float32)[0]
|
||||
)
|
||||
|
||||
|
||||
def bf_lo(w):
|
||||
"""The f32 of a packed bf16 pair word's low half."""
|
||||
return (w << 16).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf_hi(w):
|
||||
"""The f32 of a packed bf16 pair word's high half."""
|
||||
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def bf2_f32(w):
|
||||
"""Packed bf16 pair word -> (f32 low, f32 high)."""
|
||||
return bf_lo(w), bf_hi(w)
|
||||
|
||||
|
||||
def fp8_pack4(a, b, c, d):
|
||||
"""Four f32 (already in range) -> one dword of four E4M3 bytes, RNE."""
|
||||
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
|
||||
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP ranks
|
||||
def peer_bases(peers, tp):
|
||||
"""Every rank's peer buffer base from the ``peers`` table (an i64 a rank),
|
||||
loaded once: a put's address must not wait on a load of its own."""
|
||||
return [load_ptr64(peers, p) for p in range(tp)]
|
||||
|
||||
|
||||
def sum_partials(poll, own, pair_of, tp):
|
||||
"""The ``tp`` ranks' bf16 pair partials at ``pair_of(src)`` of the mailbox
|
||||
region ``own``, summed in fp32 in rank order 0 .. tp - 1: one poll batch ->
|
||||
(lo sum, hi sum). Every rank sums in the same order, so they agree."""
|
||||
ws = poll([(own, pair_of(src), 1) for src in range(tp)])
|
||||
acc0, acc1 = bf2_f32(ws[0][0])
|
||||
for src in range_constexpr(1, tp):
|
||||
lo, hi = bf2_f32(ws[src][0])
|
||||
acc0 = acc0 + lo
|
||||
acc1 = acc1 + hi
|
||||
return acc0, acc1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- timeline stamps
|
||||
def stamp(on, tls, tid, k):
|
||||
# compile-time gate outside, traced condition inside: they cannot be one `and`
|
||||
if const_expr(on): # noqa: SIM102
|
||||
if tid == 0:
|
||||
fx.ptr_store(memrealtime(), tls + k)
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/runtime/consensus.py
|
||||
# atom/mono/runtime/peer_memory.py
|
||||
"""Symmetric peer memory for the in-kernel TP all-reduces, and the agreement every
|
||||
rank must reach before anything collective (a rank that turned the mono layer
|
||||
off alone would leave the others spinning on the GPU)."""
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from aiter.ops.flydsl.quick_allreduce_int4_ipc import UncachedIpcHeap
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP agreement
|
||||
class MonoUnsupported(Exception):
|
||||
"""The loaded model or runtime configuration is outside what mono serves."""
|
||||
|
||||
|
||||
def tp_agree(ok: bool, group) -> bool:
|
||||
"""True when every rank of ``group`` passed ``ok`` true. Every rank must call
|
||||
it at the same point; ``group`` is a CPU (gloo) process group, or None for a
|
||||
single rank."""
|
||||
if group is None:
|
||||
return ok
|
||||
flag = torch.tensor([int(ok)], dtype=torch.int32)
|
||||
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group)
|
||||
return bool(flag.item())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- peer buffers
|
||||
class _DeviceBytes:
|
||||
"""``__cuda_array_interface__`` of ``nbytes`` bytes at a raw device address, for a
|
||||
non-owning torch view."""
|
||||
|
||||
def __init__(self, ptr: int, nbytes: int):
|
||||
self.__cuda_array_interface__ = {
|
||||
"shape": (nbytes,),
|
||||
"typestr": "|u1",
|
||||
"data": (ptr, False),
|
||||
"version": 3,
|
||||
}
|
||||
|
||||
|
||||
class PeerBuffer:
|
||||
"""One uncached buffer per rank, every rank holding every peer's address
|
||||
(``addresses``, int64, indexed by TP rank). What lives where, and why no rank
|
||||
overwrites a peer's unread data, is the model's layout's to say.
|
||||
|
||||
The constructor is collective over ``group`` (the handle exchange, then an
|
||||
agreement that every rank mapped every peer): every rank must reach it, so a
|
||||
runner reaches it only after ``tp_agree``. A failed mapping on any rank raises
|
||||
``MonoUnsupported`` on all of them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
nbytes: int,
|
||||
group,
|
||||
rank: int,
|
||||
npes: int,
|
||||
device: torch.device,
|
||||
):
|
||||
self.local = UncachedIpcHeap.alloc_uncached(nbytes)
|
||||
# a view, not an owner: close() frees the memory under it
|
||||
self.bytes = torch.as_tensor(_DeviceBytes(self.local, nbytes), device=device)
|
||||
self.rank = rank
|
||||
self._opened: list[int] = []
|
||||
addresses = [self.local]
|
||||
if npes > 1:
|
||||
handles = UncachedIpcHeap.gather_object_list_via_broadcast(
|
||||
group, UncachedIpcHeap.get_mem_handle_bytes(self.local)
|
||||
)
|
||||
addresses, failure = [], None
|
||||
try:
|
||||
for peer, handle in enumerate(handles):
|
||||
if peer == rank:
|
||||
addresses.append(self.local)
|
||||
else:
|
||||
base = UncachedIpcHeap.open_mem_handle(handle)
|
||||
self._opened.append(base)
|
||||
addresses.append(base)
|
||||
except RuntimeError as err: # every rank must still reach the agreement
|
||||
failure = err
|
||||
if not tp_agree(failure is None, group):
|
||||
self.close()
|
||||
raise MonoUnsupported(
|
||||
"peer memory handshake failed on "
|
||||
f"{'this' if failure else 'another'}"
|
||||
" TP rank"
|
||||
) from failure
|
||||
self.addresses = torch.tensor(addresses, dtype=torch.int64, device=device)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the peer mappings and free the local buffer; idempotent. Only once no
|
||||
kernel of any rank can still touch it (the runner is being dropped)."""
|
||||
opened, self._opened = self._opened, []
|
||||
for base in opened:
|
||||
UncachedIpcHeap.close_mem_handle(base)
|
||||
if self.local:
|
||||
self.bytes = None
|
||||
UncachedIpcHeap.free_device_mem(self.local)
|
||||
self.local = 0
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/plan/execution.py
|
||||
# atom/mono/plan/layout.py
|
||||
# atom/mono/plan/arith.py
|
||||
# atom/mono/plan/shard.py
|
||||
# atom/mono/plan/build_key.py
|
||||
"""Plain-Python planning of the mono kernels: the execution model (one CTA a CU,
|
||||
all resident), mailbox layouts, arithmetic that runs on ints and traced values
|
||||
alike, TP shards, and build keys (every build parameter, and this package's
|
||||
sources, in FlyDSL's JIT cache key)."""
|
||||
|
||||
import enum
|
||||
import functools
|
||||
import hashlib
|
||||
from dataclasses import dataclass, fields
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------- execution model
|
||||
BLOCKS = 256
|
||||
THREADS = 512
|
||||
WAVES = THREADS // 64
|
||||
|
||||
|
||||
def first_task(bid, base):
|
||||
"""CTA ``bid``'s first task of a stage placed from CTA ``base`` (any count of
|
||||
CTAs past 0: placements that add task counts run past the grid), then every
|
||||
BLOCKS-th."""
|
||||
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- mailbox layouts
|
||||
PAIR_BYTES = 8 # a (value, tag) pair
|
||||
|
||||
|
||||
def pair_layout(items, align: int = PAIR_BYTES, start: int = 0) -> dict:
|
||||
"""``items`` (name, pairs) laid out in order: name -> (byte offset, bytes),
|
||||
each region from an ``align``-byte boundary at or past ``start``."""
|
||||
out, off = {}, start
|
||||
for name, pairs in items:
|
||||
off = (off + align - 1) // align * align
|
||||
out[name] = (off, pairs * PAIR_BYTES)
|
||||
off += pairs * PAIR_BYTES
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- arithmetic
|
||||
def sel(pred, a, b):
|
||||
"""A if pred else b: a conditional on Python values, ``select`` on traced
|
||||
ones (rewrapped in the traced operand's type)."""
|
||||
if isinstance(pred, bool):
|
||||
return a if pred else b
|
||||
out = pred.select(a, b)
|
||||
traced = [x for x in (a, b) if not isinstance(x, int)]
|
||||
return type(traced[0])(out) if traced else out
|
||||
|
||||
|
||||
def imin(a, b):
|
||||
return sel(a < b, a, b)
|
||||
|
||||
|
||||
def i32(x):
|
||||
"""``x``, asserted inside the device's Int32 range when it is a Python int."""
|
||||
if isinstance(x, int):
|
||||
assert -(2**31) <= x < 2**31, x
|
||||
return x
|
||||
|
||||
|
||||
def cdiv(a, b):
|
||||
"""ceil(a / b) for a >= 0, b > 0."""
|
||||
return i32((a + b - 1) // b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TP shards
|
||||
class ShardError(ValueError):
|
||||
"""A width the TP size does not divide, or a TP size the model refuses."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Shard:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tp < 1:
|
||||
raise ShardError(f"TP {self.tp}")
|
||||
|
||||
def split(self, full: int, what: str) -> int:
|
||||
"""``full`` / tp, refusing a remainder: a rank's share of ``what``."""
|
||||
if full % self.tp:
|
||||
raise ShardError(f"{what} {full} is not divisible by TP {self.tp}")
|
||||
return full // self.tp
|
||||
|
||||
|
||||
def padded(n: int, align: int) -> int:
|
||||
return cdiv(n, align) * align
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- build keys
|
||||
_SCALARS = (int, float, bool, str, enum.Enum)
|
||||
_ROOT = Path(__file__).resolve().parents[1] # the mono package
|
||||
|
||||
|
||||
@functools.cache
|
||||
def source_digest(*paths: str) -> str:
|
||||
"""sha256 of every ``.py`` file under ``paths`` (files or directories,
|
||||
relative to the mono package), each file's path and contents: one model's
|
||||
kernel sources, so an edit to another model's rebuilds none of these."""
|
||||
h = hashlib.sha256()
|
||||
for rel in paths:
|
||||
root = _ROOT / rel
|
||||
files = [root] if root.is_file() else sorted(root.rglob("*.py"))
|
||||
if not files:
|
||||
raise FileNotFoundError(f"no kernel sources at {root}")
|
||||
for f in files:
|
||||
h.update(str(f.relative_to(_ROOT)).encode())
|
||||
h.update(f.read_bytes())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def key_tuple(key, sources: str) -> tuple:
|
||||
"""``key``'s (field name, value) pairs, every value a scalar, then
|
||||
("sources", ``sources``): the model's ``source_digest``."""
|
||||
out = []
|
||||
for f in fields(key):
|
||||
value = getattr(key, f.name)
|
||||
if not isinstance(value, _SCALARS):
|
||||
raise TypeError(
|
||||
f"{type(key).__name__}.{f.name} = {value!r}: a build parameter must be"
|
||||
" a scalar to reach the JIT cache key"
|
||||
)
|
||||
out.append((f.name, value))
|
||||
out.append(("sources", sources))
|
||||
return tuple(out)
|
||||
@@ -1,177 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/mono/device/sync.py
|
||||
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
|
||||
between GPUs (system scope), every address a named region (``Addr``)."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
CM_DEV,
|
||||
CM_SYS,
|
||||
bf16_pair,
|
||||
rsrc,
|
||||
traced,
|
||||
)
|
||||
|
||||
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
|
||||
|
||||
|
||||
@traced
|
||||
def _spin(load_all, pending):
|
||||
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
|
||||
loop keeps the loads from being hoisted."""
|
||||
v = load_all()
|
||||
while pending(v):
|
||||
rocdl.s_sleep(1)
|
||||
v = load_all()
|
||||
return v
|
||||
|
||||
|
||||
class Space(Enum):
|
||||
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
|
||||
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Addr:
|
||||
"""A mailbox region's address: the region's name and space, and the traced
|
||||
address. The space sets the hand-off's scope: device for SCRATCH, system for
|
||||
PEER."""
|
||||
|
||||
region: str
|
||||
space: Space
|
||||
value: object
|
||||
|
||||
|
||||
def sreg(base, offset, region):
|
||||
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
|
||||
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
|
||||
|
||||
|
||||
def preg(base, offset, region):
|
||||
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
|
||||
``base``."""
|
||||
return Addr(region, Space.PEER, base + fx.Int64(offset))
|
||||
|
||||
|
||||
# the store / load cache policy of a hand-off in each space
|
||||
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
|
||||
|
||||
|
||||
class Mailbox:
|
||||
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
|
||||
|
||||
Every 32-bit value is stored next to this launch's tag; a consumer
|
||||
polls the payload until all tags match, so a hand-off costs one round trip
|
||||
(no store drain, no separate flag). The mono layer's launches tag their
|
||||
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
|
||||
never carries a live tag. Every address is an ``Addr``: its space picks the
|
||||
scope.
|
||||
"""
|
||||
|
||||
def __init__(self, tag):
|
||||
self.tag = tag
|
||||
|
||||
def put(self, a, i, v):
|
||||
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
||||
rsrc(a.value),
|
||||
i * 2,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def put_bf(self, a, i, vs):
|
||||
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
|
||||
store."""
|
||||
words = []
|
||||
for j in range_constexpr(len(vs) // 2):
|
||||
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(a.value),
|
||||
i,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def put_words(self, a, i, words):
|
||||
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
|
||||
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
|
||||
not kept."""
|
||||
vals = []
|
||||
for w in words:
|
||||
vals += [fx.Int32(w), self.tag]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(vals, fx.Int32),
|
||||
rsrc(a.value),
|
||||
i * 2,
|
||||
cache_modifier=_SCOPE_CM[a.space],
|
||||
)
|
||||
|
||||
def poll(self, specs, batch=POLL_MAX):
|
||||
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
|
||||
all of one space.
|
||||
|
||||
All pairs are re-loaded together while any tag is stale, so a batch costs
|
||||
one round trip after its last producer lands. Returns the value words."""
|
||||
if const_expr(len(specs) == 0):
|
||||
return []
|
||||
if const_expr(len(specs) > batch):
|
||||
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
||||
spaces = {a.space for a, _, _ in specs}
|
||||
assert len(spaces) == 1, f"one poll batch spans {spaces}"
|
||||
cm = _SCOPE_CM[spaces.pop()]
|
||||
bases = [a.value for a, _, _ in specs]
|
||||
tag = self.tag
|
||||
|
||||
def load_all():
|
||||
words = []
|
||||
for b, (_, i, n) in zip(bases, specs):
|
||||
w = fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(b),
|
||||
fx.Int32(i) * 2,
|
||||
vec_width=2 * n,
|
||||
dtype=T.i32,
|
||||
cache_modifier=cm,
|
||||
)
|
||||
)
|
||||
words += [w[e] for e in range(2 * n)]
|
||||
return fx.Vector.from_elements(words, fx.Int32)
|
||||
|
||||
nw = sum(2 * n for _, _, n in specs)
|
||||
|
||||
def pending(v):
|
||||
bad = v[1] != tag
|
||||
for e in range_constexpr(3, nw, 2):
|
||||
bad = bad | (v[e] != tag)
|
||||
return bad
|
||||
|
||||
v = _spin(load_all, pending)
|
||||
outs, e = [], 0
|
||||
for _, _, n in specs:
|
||||
outs.append([v[e + 2 * q] for q in range(n)])
|
||||
e += 2 * n
|
||||
return outs
|
||||
|
||||
|
||||
@traced
|
||||
def publish(put, a, i, v, who):
|
||||
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
|
||||
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
|
||||
not overtake the data it announces, and a workgroup-scope release fence does
|
||||
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
|
||||
mode: the workgroup's waves share one CU, the readers here do not)."""
|
||||
rocdl.s_waitcnt(vmcnt=0)
|
||||
gpu.barrier()
|
||||
if who:
|
||||
put(a, i, v)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,464 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1-Flash mono decode layer: host side.
|
||||
|
||||
One ``DSV41MonoLayer`` a TP rank runs every mono layer of a decode step (M <= 48
|
||||
rows): two persistent launches a layer (``layer``) from the layer's
|
||||
inputs at the attention seam to its outputs at the next one -- the MoE's
|
||||
reduced output, the residual after the FFN seam and that seam's mixes -- with
|
||||
both TP all-reduces inside the kernels (symmetric peer memory). A layer whose
|
||||
attention vLLM runs (``ffn``) takes one launch from its unreduced attention
|
||||
output on. Every argument is a device pointer: the launches can be captured in a
|
||||
HIP graph. The kernels move their mailbox epoch on themselves; every rank runs
|
||||
the same launches, so the ranks' epochs agree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
|
||||
from .common.plan import BLOCKS
|
||||
from .layer import (
|
||||
EPOCH_WORDS,
|
||||
MAX_TOKENS,
|
||||
MonoBuild,
|
||||
build_mono_ffn,
|
||||
build_mono_k1,
|
||||
build_mono_k2,
|
||||
peer_half_bytes,
|
||||
scratch_bytes,
|
||||
)
|
||||
from .stages.dims import Dims as MoeDims
|
||||
from .stages.moe_shape import EXPERTS
|
||||
|
||||
__all__ = ["BLOCKS", "MAX_TOKENS", "AttnWeights", "DSV41MonoLayer", "MonoLayerWeights"]
|
||||
|
||||
_LOG2E = 1.4426950408889634
|
||||
HC = 4
|
||||
|
||||
|
||||
def _ensure_writable_flydsl_cache() -> None:
|
||||
"""Aiter points FlyDSL at its bundled, read-only cache; new kernels need a
|
||||
writable one."""
|
||||
cur = os.environ.get("FLYDSL_RUNTIME_CACHE_DIR")
|
||||
if cur and os.access(cur, os.W_OK):
|
||||
return
|
||||
path = Path.home() / ".flydsl" / "cache"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = str(path)
|
||||
|
||||
|
||||
def _check_tensors(owner, tag: str, want: dict) -> None:
|
||||
"""Each named tensor of ``owner`` has the (shape, dtype) the kernels read,
|
||||
contiguous: their buffer loads are unbounded, so a mismatch would read
|
||||
garbage rather than fault."""
|
||||
for name, (shape, dtype) in want.items():
|
||||
t = getattr(owner, name)
|
||||
assert tuple(t.shape) == shape and t.dtype == dtype and t.is_contiguous(), (
|
||||
f"{tag} {name}: {tuple(t.shape)} {t.dtype}, want {shape} {dtype}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttnWeights:
|
||||
"""One layer's attention tensors on this rank, in vLLM's loaded layout."""
|
||||
|
||||
layer_id: int
|
||||
wqkv: torch.Tensor # [1792, 5120] e4m3: [wq_a; wkv]
|
||||
wqkv_scale: torch.Tensor # [56, 160] uint8 E8M0 (32 x 32 blocks)
|
||||
q_norm: torch.Tensor # [1280] bf16
|
||||
kv_norm: torch.Tensor # [512] bf16
|
||||
wq_b: torch.Tensor # [H * 512, 1280] e4m3
|
||||
wq_b_scale: torch.Tensor # [H * 16, 40] uint8
|
||||
wo_a: torch.Tensor # [G * 1024, 4096] e4m3
|
||||
wo_a_scale: torch.Tensor # [G * 32, 128] uint8
|
||||
wo_b: torch.Tensor # [5120, G * 1024] e4m3
|
||||
wo_b_scale: torch.Tensor # [160, G * 32] uint8
|
||||
attn_sink: torch.Tensor # [>= H] f32
|
||||
cos_sin: torch.Tensor # [positions, 64] f32 (the layer's rope table)
|
||||
ratio: int # 0 / 1 / 2: the layer's compress ratio
|
||||
|
||||
def check(self, d: Dims) -> None:
|
||||
h, g = d.heads, d.groups
|
||||
want = {
|
||||
"wqkv": ((1792, HIDDEN), torch.float8_e4m3fn),
|
||||
"wqkv_scale": ((56, HIDDEN // 32), torch.uint8),
|
||||
"wq_b": ((h * HEAD_DIM, 1280), torch.float8_e4m3fn),
|
||||
"wq_b_scale": ((h * HEAD_DIM // 32, 40), torch.uint8),
|
||||
"wo_a": ((g * 1024, d.group_k), torch.float8_e4m3fn),
|
||||
"wo_a_scale": ((g * 32, d.group_k // 32), torch.uint8),
|
||||
"wo_b": ((HIDDEN, g * 1024), torch.float8_e4m3fn),
|
||||
"wo_b_scale": ((HIDDEN // 32, g * 32), torch.uint8),
|
||||
}
|
||||
_check_tensors(self, f"layer {self.layer_id}", want)
|
||||
assert self.cos_sin.dtype == torch.float32 and self.cos_sin.shape[-1] == 64
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonoLayerWeights:
|
||||
"""One layer's tensors on this rank, in vLLM's loaded layout (``attn``:
|
||||
None for a layer whose attention vLLM runs)."""
|
||||
|
||||
attn: AttnWeights | None
|
||||
hc_attn_fn: torch.Tensor # [24, 4 * 5120] f32
|
||||
hc_attn_scale: torch.Tensor # [3] f32
|
||||
hc_attn_base: torch.Tensor # [24] f32
|
||||
attn_norm: torch.Tensor # [5120] bf16
|
||||
hc_ffn_fn: torch.Tensor
|
||||
hc_ffn_scale: torch.Tensor
|
||||
hc_ffn_base: torch.Tensor
|
||||
ffn_norm: torch.Tensor
|
||||
gate_w: torch.Tensor # [384, 5120] bf16
|
||||
bias: torch.Tensor # [384] f32 (e_score_correction_bias)
|
||||
w13: torch.Tensor # [384, 2 inter, 2560] fp4x2, aiter (16, 16) shuffle
|
||||
w13_s: torch.Tensor # its scales, aiter shuffle_scale order
|
||||
w2: torch.Tensor # [384, 5120, inter / 2] fp4x2
|
||||
w2_s: torch.Tensor
|
||||
sgu: torch.Tensor # shared gate_up [2 inter, 5120] e4m3, row-major
|
||||
sgu_s: torch.Tensor # [2 inter / 32, 160] E8M0
|
||||
sw2: torch.Tensor # shared down [5120, inter] e4m3
|
||||
sw2_s: torch.Tensor
|
||||
|
||||
def check(self, tp: int) -> None:
|
||||
"""The MoE's tensors: 384 experts, TP-sharded intermediates (the routed
|
||||
one padded as the loader pads it), AITER's A8W4 layout, 32 x 32 E8M0
|
||||
blocks for the shared expert."""
|
||||
d, fp4, e4m3, u8 = (
|
||||
MoeDims(tp),
|
||||
torch.float4_e2m1fn_x2,
|
||||
torch.float8_e4m3fn,
|
||||
torch.uint8,
|
||||
)
|
||||
want = {
|
||||
"gate_w": ((EXPERTS, HIDDEN), torch.bfloat16),
|
||||
"bias": ((EXPERTS,), torch.float32),
|
||||
"w13": ((EXPERTS, 2 * d.inter, HIDDEN // 2), fp4),
|
||||
"w13_s": ((EXPERTS * 2 * d.inter, HIDDEN // 32), u8),
|
||||
"w2": ((EXPERTS, HIDDEN, d.inter // 2), fp4),
|
||||
"w2_s": ((EXPERTS * HIDDEN, d.down_scale_cols), u8),
|
||||
"sgu": ((2 * d.sh_inter, HIDDEN), e4m3),
|
||||
"sgu_s": ((2 * d.sh_inter // 32, HIDDEN // 32), u8),
|
||||
"sw2": ((HIDDEN, d.sh_inter), e4m3),
|
||||
"sw2_s": ((HIDDEN // 32, d.sh_inter // 32), u8),
|
||||
}
|
||||
_check_tensors(self, "MoE", want)
|
||||
|
||||
|
||||
class DSV41MonoLayer:
|
||||
"""The mono decode runner of one TP rank (``group``: its TP group, for the
|
||||
peer-memory handle exchange; a gloo / CPU group)."""
|
||||
|
||||
def __init__(self, tp: int, rank: int, group, device: torch.device | str = "cuda"):
|
||||
_ensure_writable_flydsl_cache()
|
||||
from .common.peer_memory import PeerBuffer
|
||||
|
||||
self.tp, self.rank = tp, rank
|
||||
self.d = Dims(tp)
|
||||
dev = self.device = torch.device(device)
|
||||
self._scratch: dict[int, torch.Tensor] = {}
|
||||
# [epoch, -, -, -, a mark per CTA, the MoE's counters]
|
||||
self.epoch = torch.zeros(EPOCH_WORDS, dtype=torch.int32, device=dev)
|
||||
self.q = torch.zeros(
|
||||
MAX_TOKENS, self.d.heads, HEAD_DIM, dtype=torch.bfloat16, device=dev
|
||||
)
|
||||
self.kt = torch.zeros(MAX_TOKENS * KEYS, dtype=torch.int32, device=dev)
|
||||
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
|
||||
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
|
||||
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
|
||||
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
|
||||
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self._first_pre[:, 0] = 1.0
|
||||
# an index source's K1 outputs, vLLM's indexer's inputs
|
||||
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
|
||||
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
|
||||
# the attention seam's outputs, K1 -> K2
|
||||
self.res_mid = torch.zeros(
|
||||
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
|
||||
)
|
||||
self.post_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self.comb_a = torch.zeros(MAX_TOKENS, HC, HC, dtype=torch.float32, device=dev)
|
||||
self.pre_a = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self.peer = PeerBuffer(2 * peer_half_bytes(tp), group, rank, tp, dev)
|
||||
self.peer.bytes.zero_()
|
||||
self._qk_scale = (
|
||||
torch.tensor([HEAD_DIM**-0.5 * _LOG2E], dtype=torch.float32)
|
||||
.view(torch.int32)
|
||||
.item()
|
||||
)
|
||||
self._kernels: dict = {}
|
||||
|
||||
def scratch(self, tokens: int) -> torch.Tensor:
|
||||
"""Step width ``tokens``'s scratch: a width's layout has memory of its own
|
||||
(``layer.scratch_layout``). Allocated at the width's first step, which is
|
||||
eager: vLLM runs every graph's batch eagerly before capturing it."""
|
||||
buf = self._scratch.get(tokens)
|
||||
if buf is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
f"DSv4.1 mono decode: step width {tokens} first reached inside "
|
||||
"a CUDA graph capture"
|
||||
)
|
||||
buf = torch.zeros(
|
||||
scratch_bytes(tokens, self.tp), dtype=torch.uint8, device=self.device
|
||||
)
|
||||
self._scratch[tokens] = buf
|
||||
return buf
|
||||
|
||||
def first_pre_mix(self, tokens: int) -> torch.Tensor:
|
||||
return self._first_pre[:tokens]
|
||||
|
||||
@staticmethod
|
||||
def supports(tokens: int) -> bool:
|
||||
return 1 <= tokens <= MAX_TOKENS
|
||||
|
||||
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
|
||||
key = (tokens, ratio, fold, index)
|
||||
if key not in self._kernels:
|
||||
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
|
||||
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
|
||||
return self._kernels[key]
|
||||
|
||||
def ffn_kernel(self, tokens: int):
|
||||
key = (tokens, "ffn")
|
||||
if key not in self._kernels:
|
||||
self._kernels[key] = build_mono_ffn(MonoBuild(tokens, self.tp, 0))
|
||||
return self._kernels[key]
|
||||
|
||||
@staticmethod
|
||||
def _outs(M: int, residual: torch.Tensor) -> tuple:
|
||||
dev = residual.device
|
||||
return (
|
||||
torch.empty(M, HIDDEN, dtype=torch.bfloat16, device=dev),
|
||||
torch.empty_like(residual),
|
||||
torch.empty(M, HC, 1, dtype=torch.float32, device=dev),
|
||||
torch.empty(M, HC, HC, dtype=torch.float32, device=dev),
|
||||
torch.empty(M, HC, dtype=torch.float32, device=dev),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
w: MonoLayerWeights,
|
||||
x: torch.Tensor, # [M, 5120] bf16: the previous FFN output (reduced)
|
||||
residual: torch.Tensor, # [M, 4, 5120] bf16
|
||||
post_mix: torch.Tensor, # [M, 4, 1] f32
|
||||
res_mix: torch.Tensor, # [M, 4, 4] f32
|
||||
pre_mix: torch.Tensor, # [M, 4] f32
|
||||
positions: torch.Tensor, # [M] int64
|
||||
slot_mapping: torch.Tensor, # [M] int64
|
||||
swa_cache: torch.Tensor, # [blocks, block, 584] uint8
|
||||
swa_indices: torch.Tensor, # [>= M, 1, 128] int32
|
||||
swa_lens: torch.Tensor, # [>= M] int32
|
||||
token_to_req: torch.Tensor, # [>= M] int32
|
||||
topk_indices: torch.Tensor | None = None,
|
||||
comp_cache: torch.Tensor | None = None,
|
||||
comp_block_table: torch.Tensor | None = None,
|
||||
outs: tuple | None = None,
|
||||
fold: bool = True,
|
||||
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
|
||||
):
|
||||
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
|
||||
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
|
||||
source's top-k selection into ``topk_indices``, run between the
|
||||
kernels on K1's normed input and qr."""
|
||||
M = x.shape[0]
|
||||
assert self.supports(M), M
|
||||
a = w.attn
|
||||
assert a is not None
|
||||
ratio = a.ratio
|
||||
index = select is not None
|
||||
assert not index or ratio
|
||||
k1, k2 = self.kernels(M, ratio, fold, index)
|
||||
st = torch.cuda.current_stream()
|
||||
if outs is None:
|
||||
outs = self._outs(M, residual)
|
||||
out, res_out, post_out, comb_out, pre_out = outs
|
||||
for t_ in (x, residual, post_mix, res_mix, pre_mix, positions, slot_mapping):
|
||||
assert t_.is_contiguous()
|
||||
assert positions.dtype == slot_mapping.dtype == torch.int64
|
||||
assert swa_indices.dtype == swa_lens.dtype == token_to_req.dtype == torch.int32
|
||||
assert swa_indices.stride(0) == 128 and swa_cache.dtype == torch.uint8
|
||||
assert swa_cache.shape[-1] == 584 and swa_cache.stride(1) == 584
|
||||
swa_block = swa_cache.shape[1]
|
||||
if ratio:
|
||||
assert topk_indices is not None and comp_cache is not None
|
||||
assert comp_block_table is not None
|
||||
assert topk_indices.dtype == torch.int32 and topk_indices.stride(0) == 512
|
||||
assert comp_cache.dtype == torch.uint8 and comp_cache.stride(1) == 584
|
||||
assert comp_block_table.dtype == torch.int32
|
||||
comp, comp_stride, comp_block = (
|
||||
comp_cache,
|
||||
comp_cache.stride(0),
|
||||
comp_cache.shape[1],
|
||||
)
|
||||
bt, bt_stride, topk = (
|
||||
comp_block_table,
|
||||
comp_block_table.stride(0),
|
||||
topk_indices,
|
||||
)
|
||||
else:
|
||||
comp, comp_stride, comp_block = self._dummy, 0, 1
|
||||
bt, bt_stride, topk = self._dummy, 0, self._dummy
|
||||
k1(
|
||||
residual.data_ptr(),
|
||||
x.data_ptr(),
|
||||
post_mix.data_ptr(),
|
||||
res_mix.data_ptr(),
|
||||
pre_mix.data_ptr(),
|
||||
w.hc_attn_fn.data_ptr(),
|
||||
w.hc_attn_scale.data_ptr(),
|
||||
w.hc_attn_base.data_ptr(),
|
||||
w.attn_norm.data_ptr(),
|
||||
self.res_mid.data_ptr(),
|
||||
self.post_a.data_ptr(),
|
||||
self.comb_a.data_ptr(),
|
||||
self.pre_a.data_ptr(),
|
||||
a.wqkv.data_ptr(),
|
||||
a.wqkv_scale.data_ptr(),
|
||||
a.q_norm.data_ptr(),
|
||||
a.kv_norm.data_ptr(),
|
||||
a.wq_b.data_ptr(),
|
||||
a.wq_b_scale.data_ptr(),
|
||||
a.cos_sin.data_ptr(),
|
||||
positions.data_ptr(),
|
||||
slot_mapping.data_ptr(),
|
||||
swa_cache.data_ptr(),
|
||||
swa_cache.stride(0),
|
||||
swa_block,
|
||||
self.q.data_ptr(),
|
||||
swa_indices.data_ptr(),
|
||||
swa_lens.data_ptr(),
|
||||
comp_block,
|
||||
bt.data_ptr(),
|
||||
bt_stride,
|
||||
topk.data_ptr(),
|
||||
token_to_req.data_ptr(),
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
self.normed.data_ptr(),
|
||||
self.qr.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.epoch.data_ptr(),
|
||||
0,
|
||||
stream=st,
|
||||
)
|
||||
if select is not None:
|
||||
select(self.normed[:M], self.qr[:M])
|
||||
k2(
|
||||
self.q.data_ptr(),
|
||||
swa_cache.data_ptr(),
|
||||
swa_cache.stride(0),
|
||||
swa_block,
|
||||
comp.data_ptr(),
|
||||
comp_stride,
|
||||
comp_block,
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
slot_mapping.data_ptr(),
|
||||
swa_indices.data_ptr(),
|
||||
swa_lens.data_ptr(),
|
||||
token_to_req.data_ptr(),
|
||||
topk.data_ptr(),
|
||||
bt.data_ptr(),
|
||||
bt_stride,
|
||||
positions.data_ptr(),
|
||||
a.attn_sink.data_ptr(),
|
||||
self._qk_scale,
|
||||
a.cos_sin.data_ptr(),
|
||||
a.wo_a.data_ptr(),
|
||||
a.wo_a_scale.data_ptr(),
|
||||
a.wo_b.data_ptr(),
|
||||
a.wo_b_scale.data_ptr(),
|
||||
self._zero_rec.data_ptr(),
|
||||
self.res_mid.data_ptr(),
|
||||
self.post_a.data_ptr(),
|
||||
self.comb_a.data_ptr(),
|
||||
self.pre_a.data_ptr(),
|
||||
w.hc_ffn_fn.data_ptr(),
|
||||
w.hc_ffn_scale.data_ptr(),
|
||||
w.hc_ffn_base.data_ptr(),
|
||||
w.ffn_norm.data_ptr(),
|
||||
res_out.data_ptr(),
|
||||
post_out.data_ptr(),
|
||||
comb_out.data_ptr(),
|
||||
pre_out.data_ptr(),
|
||||
w.gate_w.data_ptr(),
|
||||
w.bias.data_ptr(),
|
||||
w.w13.data_ptr(),
|
||||
w.w13_s.data_ptr(),
|
||||
w.w2.data_ptr(),
|
||||
w.w2_s.data_ptr(),
|
||||
w.sgu.data_ptr(),
|
||||
w.sgu_s.data_ptr(),
|
||||
w.sw2.data_ptr(),
|
||||
w.sw2_s.data_ptr(),
|
||||
out.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.peer.local,
|
||||
self.peer.addresses.data_ptr(),
|
||||
self.rank,
|
||||
self.epoch.data_ptr(),
|
||||
0,
|
||||
stream=st,
|
||||
)
|
||||
return outs
|
||||
|
||||
def ffn(
|
||||
self,
|
||||
w: MonoLayerWeights,
|
||||
part: torch.Tensor, # [M, 5120] bf16: this rank's unreduced wo_b output
|
||||
residual: torch.Tensor, # [M, 4, 5120] bf16, after the attention seam
|
||||
post_mix: torch.Tensor, # [M, 4, 1] f32, the attention seam's
|
||||
res_mix: torch.Tensor, # [M, 4, 4] f32
|
||||
pre_mix: torch.Tensor, # [M, 4] f32
|
||||
outs: tuple | None = None,
|
||||
):
|
||||
"""-> (out, residual, post_mix, res_mix, pre_mix) of a layer whose
|
||||
attention vLLM ran: the attention's TP reduce, the FFN seam, the MoE and
|
||||
its all-reduce in one launch."""
|
||||
M = part.shape[0]
|
||||
assert self.supports(M), M
|
||||
for t_ in (part, residual, post_mix, res_mix, pre_mix):
|
||||
assert t_.is_contiguous()
|
||||
assert part.dtype == residual.dtype == torch.bfloat16
|
||||
if outs is None:
|
||||
outs = self._outs(M, residual)
|
||||
out, res_out, post_out, comb_out, pre_out = outs
|
||||
self.ffn_kernel(M)(
|
||||
part.data_ptr(),
|
||||
residual.data_ptr(),
|
||||
post_mix.data_ptr(),
|
||||
res_mix.data_ptr(),
|
||||
pre_mix.data_ptr(),
|
||||
w.hc_ffn_fn.data_ptr(),
|
||||
w.hc_ffn_scale.data_ptr(),
|
||||
w.hc_ffn_base.data_ptr(),
|
||||
w.ffn_norm.data_ptr(),
|
||||
res_out.data_ptr(),
|
||||
post_out.data_ptr(),
|
||||
comb_out.data_ptr(),
|
||||
pre_out.data_ptr(),
|
||||
w.gate_w.data_ptr(),
|
||||
w.bias.data_ptr(),
|
||||
w.w13.data_ptr(),
|
||||
w.w13_s.data_ptr(),
|
||||
w.w2.data_ptr(),
|
||||
w.w2_s.data_ptr(),
|
||||
w.sgu.data_ptr(),
|
||||
w.sgu_s.data_ptr(),
|
||||
w.sw2.data_ptr(),
|
||||
w.sw2_s.data_ptr(),
|
||||
out.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.peer.local,
|
||||
self.peer.addresses.data_ptr(),
|
||||
self.rank,
|
||||
self.epoch.data_ptr(),
|
||||
stream=torch.cuda.current_stream(),
|
||||
)
|
||||
return outs
|
||||
@@ -1,8 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The mono layer's kernel sources, in every build's JIT cache key
|
||||
(``common.plan.key_tuple``): an edit anywhere in this package rebuilds."""
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import source_digest
|
||||
|
||||
SOURCES = source_digest(".")
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The V4.1 stages the mono layer adapts from ROCm/ATOM: the mHC seams
|
||||
(``seam``), the MoE (``moe``) and the GEMV / LDS helpers they share (``gemv``)."""
|
||||
@@ -1,60 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/dims.py
|
||||
"""A TP rank's share of V4.1's MoE widths: every constant the MoE stages derive
|
||||
from the TP size, in one place (a build key's ``tp`` picks it)."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import Shard, padded
|
||||
|
||||
# the model's full widths
|
||||
MOE_INTER = 2304 # a routed expert's intermediate width
|
||||
SHARED_INTER = 2304 # the shared expert's
|
||||
# the loader pads a routed expert's rank share to a multiple of this (``FusedMoE``
|
||||
# ``pad_align``) with zero weights
|
||||
MOE_PAD = 128
|
||||
ROWS = 16 # GEMV rows a task
|
||||
UG_PART = 32 # intermediate columns an ug task part: one FP4 group
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dims:
|
||||
tp: int
|
||||
|
||||
def __post_init__(self):
|
||||
shard = Shard(self.tp)
|
||||
shard.split(MOE_INTER, "routed intermediate")
|
||||
shard.split(SHARED_INTER, "shared intermediate")
|
||||
|
||||
@property
|
||||
def inter_real(self) -> int:
|
||||
"""A routed expert's real intermediate width on this rank."""
|
||||
return MOE_INTER // self.tp
|
||||
|
||||
@property
|
||||
def inter(self) -> int:
|
||||
"""... padded as the loader pads it (the rest zero weights)."""
|
||||
return padded(self.inter_real, MOE_PAD)
|
||||
|
||||
@property
|
||||
def sh_inter(self) -> int:
|
||||
"""The shared expert's intermediate width on this rank."""
|
||||
return SHARED_INTER // self.tp
|
||||
|
||||
@property
|
||||
def shared_tasks(self) -> int:
|
||||
return self.sh_inter // ROWS
|
||||
|
||||
@property
|
||||
def mid_words(self) -> int:
|
||||
"""MXFP8 words of a pick's routed intermediate."""
|
||||
return self.inter // 4
|
||||
|
||||
@property
|
||||
def down_scale_cols(self) -> int:
|
||||
"""w2_s's e8m0 columns: inter / 32, padded to a multiple of 8
|
||||
(aiter ``shuffle_scale``)."""
|
||||
return padded(self.inter // 32, 8)
|
||||
@@ -1,294 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (its GEMV and LDS helpers).
|
||||
"""The FP8 GEMV tiles of the seam and MoE stages and the helpers they share:
|
||||
token tiles, LDS rows of MXFP8 words, scalar loads and the mailbox-to-LDS
|
||||
copy."""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.mx import (
|
||||
UNIT_SCALE,
|
||||
mfma_scaled,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
CM_NT,
|
||||
fresh,
|
||||
rsrc,
|
||||
traced,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
|
||||
from vllm.models.deepseek_v41.amd.mono.common.sync import POLL_MAX
|
||||
|
||||
ROWS = 16 # GEMV rows a task
|
||||
TILE = 16 # an MFMA's N: the tokens a GEMV pass takes (``token_tiles``)
|
||||
|
||||
|
||||
def ld_f32(ptr, i):
|
||||
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
|
||||
|
||||
|
||||
def ld_bf(ptr, i):
|
||||
return fx.Float32(
|
||||
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
||||
)
|
||||
|
||||
|
||||
def token_tiles(s):
|
||||
"""A step's tokens in GEMV passes: [(first token, tokens)]. A GEMV loads its
|
||||
weights once and runs every pass on them, an x tile in LDS at a time."""
|
||||
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
|
||||
|
||||
|
||||
def tile_rows(s):
|
||||
"""The LDS rows an x tile holds."""
|
||||
return min(s, TILE)
|
||||
|
||||
|
||||
# a token's LDS row of MXFP8 words / E8M0 codes, 16 of which a MFMA B operand
|
||||
# gathers, is padded by 16 B: at a multiple of 64 words they share a bank group
|
||||
LDS_PAD = 4
|
||||
|
||||
|
||||
def lds_row(words):
|
||||
return words + LDS_PAD
|
||||
|
||||
|
||||
def lds_at(u, row):
|
||||
"""Element u of a span of rows ``row`` wide, at the ``lds_row`` stride."""
|
||||
return u + u // row * LDS_PAD
|
||||
|
||||
|
||||
def lds_tail(k):
|
||||
"""Words a K = ``k`` row's last K step reads past its ``lds_row`` (selected
|
||||
out): the last row's buffer must hold them."""
|
||||
return max(0, 32 * -(-k // 128) - lds_row(k // 4))
|
||||
|
||||
|
||||
def plus(t0, v):
|
||||
"""Token ``v`` of the tile from ``t0`` (a first tile adds nothing; ``t0``
|
||||
traced: a tile a CTA picks at run time)."""
|
||||
return v if isinstance(t0, int) and t0 == 0 else v + t0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- GEMV
|
||||
def _fp8_step_load(c, r_w, r_ws, k, rg, st, row_major=False):
|
||||
"""K step st's global operands of rows 16 rg ..: the two 64-column weight
|
||||
chunks (the upper clamped into the row) and the raw scale word. The (16, 16)
|
||||
shuffle lays a row group out in 32-column blocks of 16 rows, a chunk two of
|
||||
them (lanes 0-31, 32-63); ``k`` % 64 = 32 (a last quarter step): the lanes
|
||||
of a block past the row read its last block. ``row_major`` (vLLM's dense
|
||||
layout, k % 64 == 0): the same registers from a row-major [N, K] weight --
|
||||
lane l row 16 rg + l % 16, K bytes 128 st + 16 (l // 16) .. and + 64."""
|
||||
lane = c["lane"]
|
||||
j = lane // 16
|
||||
kg = k // 32
|
||||
if const_expr(row_major):
|
||||
assert k % 64 == 0
|
||||
row = rg * 16 + lane % 16
|
||||
lo_at = row * (k // 4) + st * 32 + j * 4
|
||||
hi_at = row * (k // 4) + fx.min(2 * st + 1, k // 64 - 1) * 16 + j * 4
|
||||
lo = fx.Vector(
|
||||
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
hi = fx.Vector(
|
||||
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
if const_expr(k % 64 == 0):
|
||||
lo_at = (rg * (k // 64) + 2 * st) * 256 + lane * 4
|
||||
else:
|
||||
lo_at = rg * (k * 4) + fx.min(4 * st + lane // 32, kg - 1) * 128 + lane % 32 * 4
|
||||
lo = fx.Vector(
|
||||
bo.buffer_load(r_w, lo_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
if const_expr(k % 64 == 0):
|
||||
hi_at = (rg * (k // 64) + fx.min(2 * st + 1, k // 64 - 1)) * 256 + lane * 4
|
||||
else:
|
||||
hi_at = rg * (k * 4) + fx.min(2 * st + 1, k // 64 - 1) * 256 + lane * 4
|
||||
hi = fx.Vector(
|
||||
bo.buffer_load(r_w, hi_at, vec_width=4, dtype=T.i32, cache_modifier=CM_NT)
|
||||
)
|
||||
wsi = (rg // 2) * kg + fx.min(st * 4 + j, kg - 1)
|
||||
word = fx.Int32(bo.buffer_load(r_ws, wsi // 4, vec_width=1, dtype=T.i32))
|
||||
return lo, hi, (word >> (wsi % 4 * 8)) & 0xFF
|
||||
|
||||
|
||||
def _fp8_step_mfma(c, k, xl, xsl, st, ops, acc, live=None, rows=None):
|
||||
"""``acc`` through K step st's scaled MFMA: ``_fp8_step_load``'s operands
|
||||
against every token's MXFP8 row in LDS. ``live`` false: a step past the end
|
||||
(its operands those of the last step), zero at unit scale. ``k`` % 64 = 32:
|
||||
a lane's lower block past the row is zero on both sides."""
|
||||
s, lane = c["S"] if rows is None else rows, c["lane"]
|
||||
lo, hi, sa = ops
|
||||
kg = k // 32
|
||||
x_row, s_row = lds_row(k // 4), lds_row(kg)
|
||||
j = lane // 16
|
||||
col = fx.min(lane % 16, s - 1)
|
||||
if const_expr(live is not None):
|
||||
st = fx.min(st, (k + 127) // 128 - 1)
|
||||
lo = [live.select(lo[d], fx.Int32(0)) for d in range(4)]
|
||||
past = (2 * st + 1) * 64 >= k
|
||||
if const_expr(live is not None):
|
||||
past = past | ~live
|
||||
av = fx.Vector.from_elements(
|
||||
[lo[d] for d in range(4)] + [past.select(fx.Int32(0), hi[d]) for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
grp = st * 4 + j
|
||||
xlo = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
xhi = fx.Vector(
|
||||
fx.ptr_load(
|
||||
xl + (col * x_row + st * 32 + 16 + j * 4),
|
||||
result_type=fx.Vector.make_type(4, fx.Int32),
|
||||
)
|
||||
)
|
||||
if const_expr(k % 64 != 0):
|
||||
over = 4 * st + j // 2 >= kg
|
||||
av = fx.Vector.from_elements(
|
||||
[over.select(fx.Int32(0), av[d]) for d in range(4)]
|
||||
+ [av[4 + d] for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
xlo = [over.select(fx.Int32(0), xlo[d]) for d in range(4)]
|
||||
xv = fx.Vector.from_elements(
|
||||
[xlo[d] for d in range(4)]
|
||||
+ [past.select(fx.Int32(0), xhi[d]) for d in range(4)],
|
||||
fx.Int32,
|
||||
)
|
||||
sb = fx.ptr_load(xsl + (col * s_row + fx.min(grp, kg - 1)))
|
||||
unit = grp >= kg
|
||||
if const_expr(live is not None):
|
||||
unit = unit | ~live
|
||||
sa = unit.select(fx.Int32(UNIT_SCALE), sa)
|
||||
sb = unit.select(fx.Int32(UNIT_SCALE), sb)
|
||||
return mfma_scaled(av, xv, acc, sa, sb)
|
||||
|
||||
|
||||
def _gemv_steps(c, k, split, tiled=False):
|
||||
"""This wave's K steps (a step past the end marked, ``split`` waves past
|
||||
split repeat the last one's; ``tiled``: wave w takes part w % split of tile
|
||||
w / split): [(step, live)]."""
|
||||
wave = c["wave"]
|
||||
steps = (k + 127) // 128
|
||||
if const_expr(split is None):
|
||||
per = (steps + WAVES - 1) // WAVES
|
||||
ragged = steps % WAVES != 0
|
||||
return [
|
||||
(wave + WAVES * i, (wave + WAVES * i < steps) if ragged else None)
|
||||
for i in range(per)
|
||||
]
|
||||
assert split <= WAVES and steps % split == 0
|
||||
per = steps // split
|
||||
part = wave % split if tiled else fx.min(wave, split - 1)
|
||||
return [(part * per + i, None) for i in range(per)]
|
||||
|
||||
|
||||
def gemv_fp8_loads(c, w, ws, k, rg, split=None, tiled=False, row_major=False):
|
||||
"""The weight operands of ``gemv_fp8``'s rows for this wave, every one in
|
||||
flight at once (a step's latency each would put them in series): loads
|
||||
only. A step past the end loads the last step. ``tiled`` (split x tiles =
|
||||
WAVES): row groups rg .., one a group of ``split`` waves."""
|
||||
steps = (k + 127) // 128
|
||||
r_w, r_ws = rsrc(w), rsrc(ws)
|
||||
if const_expr(tiled):
|
||||
assert WAVES % split == 0
|
||||
rg = rg + c["wave"] // split
|
||||
return [
|
||||
_fp8_step_load(
|
||||
c,
|
||||
r_w,
|
||||
r_ws,
|
||||
k,
|
||||
rg,
|
||||
st if live is None else fx.min(st, steps - 1),
|
||||
row_major,
|
||||
)
|
||||
for st, live in _gemv_steps(c, k, split, tiled)
|
||||
]
|
||||
|
||||
|
||||
def gemv_fp8_mfmas(c, k, xl, xsl, red, ops, split=None, tiled=False, rows=None):
|
||||
"""``gemv_fp8_loads``' operands against the rows in LDS (``rows``: a token
|
||||
tile's, every token's by default) ->
|
||||
``red[wave][lane]``; the scaled 16x16x128 MFMA, chunk kc in dwords 0-3 and kc +
|
||||
1 in 4-7. ``k`` a multiple of 32: a last half step has its upper chunk zeroed
|
||||
on both sides at unit scale (a NaN byte past the row must not reach the
|
||||
MFMA), its loads clamped into the row: the last row group's would read past
|
||||
the tensor; a last quarter step its lower chunk's upper block too. A step
|
||||
past the end adds zero at unit scale.
|
||||
|
||||
The K order is the original kernel's, which ``row_sum`` completes:
|
||||
wave w takes steps w, w + 8, ... (``split`` None: aiter's preshuffled group32
|
||||
GEMM, P0.5 microbenchmark 1), or waves 0 .. split - 1 one contiguous K range
|
||||
each, the other waves zero (aiter's split-K bmm, whose last split sums the
|
||||
partials in split order)."""
|
||||
lane, wave = c["lane"], c["wave"]
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i, (st, live) in enumerate(_gemv_steps(c, k, split, tiled)):
|
||||
acc = _fp8_step_mfma(c, k, xl, xsl, st, ops[i], acc, live, rows)
|
||||
if const_expr(split is not None and not tiled):
|
||||
acc = fx.Vector.from_elements(
|
||||
[(wave < split).select(acc[e], fx.Float32(0.0)) for e in range(4)],
|
||||
fx.Float32,
|
||||
)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- mailbox -> LDS
|
||||
@traced
|
||||
def poll_copy(c, idx, width, spans, live=None):
|
||||
"""Mailbox spans [(region, pair_of, n, dst, row)] into LDS: element u (pair
|
||||
``pair_of(u)``) to dst + u, or ``lds_at(u, row)`` for a span of rows
|
||||
``row`` words wide. Thread ``idx`` of ``width`` takes u = idx, idx +
|
||||
width, ..., POLL_MAX a round trip, each batch stored before the next is
|
||||
polled (holding them all spills past S = 16); a thread past a span's end
|
||||
re-reads its last element and stores nothing.
|
||||
|
||||
``live``: a span's elements the reader needs (traced, <= n), a span each;
|
||||
a batch past them is not polled and an element past them neither polled
|
||||
nor written (one past a step's rows is never written)."""
|
||||
idx = fresh(idx) # its elements' indices not hoisted out of a task loop
|
||||
if const_expr(live is None):
|
||||
_poll_store_batches(
|
||||
c,
|
||||
[
|
||||
(region, pair_of, n, (dst, row), idx + width * i)
|
||||
for region, pair_of, n, dst, row in spans
|
||||
for i in range((n + width - 1) // width)
|
||||
],
|
||||
)
|
||||
return
|
||||
for (region, pair_of, n, dst, row), need in zip(spans, live):
|
||||
chunks = [
|
||||
(region, pair_of, need, (dst, row), idx + width * i)
|
||||
for i in range((n + width - 1) // width)
|
||||
]
|
||||
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
|
||||
if chunks[b0][4] < need:
|
||||
_poll_store_batches(c, chunks[b0 : b0 + POLL_MAX])
|
||||
|
||||
|
||||
@traced
|
||||
def _poll_store_batches(c, chunks):
|
||||
"""``poll_copy``'s chunks [(region, pair_of, n, (dst, row), u)], POLL_MAX a
|
||||
round trip, each batch stored before the next is polled."""
|
||||
for b0 in range_constexpr(0, len(chunks), POLL_MAX):
|
||||
batch = chunks[b0 : b0 + POLL_MAX]
|
||||
got = c["poll"]([(r, p(fx.min(u, n - 1)), 1) for r, p, n, _, u in batch])
|
||||
for (_, _, n, (dst, row), u), v in zip(batch, got):
|
||||
at = u if row is None else lds_at(u, row)
|
||||
if u < n:
|
||||
fx.ptr_store(v[0], dst + at)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/moe_plan.py
|
||||
"""Every per-step decision of which MoE ug flags (UGF) and intermediate rows
|
||||
(MID) the ug stage writes and the down stage reads, defined once.
|
||||
|
||||
The route table (``stage_route``) gives each expert with picks a slot of U, its
|
||||
picks a run from ``slot_first``, and, past one token tile, its picks tiles of
|
||||
``TILE`` from ``tile_first`` (each tile's slot ``tile_slot``). An ug task covers
|
||||
a unit -- a slot, or past one token tile a pick tile -- and one of ``ug_parts`` parts
|
||||
of the intermediate; its flag is ``ug_flag``. A down round polls each of its
|
||||
slots' flags (``polled_tile`` its tiles) before it reads their MID rows. Called
|
||||
by the kernel on traced Int32 and by the CPU property test on Python ints.
|
||||
"""
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import cdiv, imin
|
||||
|
||||
TILE = 16 # picks a pick tile: an MFMA's N
|
||||
|
||||
|
||||
def pick_tiles(count):
|
||||
"""A slot's pick tiles: its ``count`` picks, ``TILE`` a tile."""
|
||||
return cdiv(count, TILE)
|
||||
|
||||
|
||||
def ug_parts(groups, ug_groups):
|
||||
"""A unit's ug parts: the expert's ``groups`` FP4 groups of its real width,
|
||||
``ug_groups`` a part. A part wholly in the padding is not run: MID's padded
|
||||
columns stay the step's zeros, against zero weights in down."""
|
||||
return cdiv(groups, ug_groups)
|
||||
|
||||
|
||||
def ug_flag(unit, part, ug_parts):
|
||||
"""The UGF pair of ug task (``unit``, ``part``), ``ug_parts`` parts a unit."""
|
||||
return unit * ug_parts + part
|
||||
|
||||
|
||||
def ug_tasks(units, ug_parts):
|
||||
"""A step's ug tasks: every part of each of its ``units``."""
|
||||
return units * ug_parts
|
||||
|
||||
|
||||
def tile_pick0(slot_first, tile, tile_first):
|
||||
"""Pick tile ``tile``'s first pick, its slot's picks from ``slot_first`` and
|
||||
tiles from ``tile_first``."""
|
||||
return slot_first + (tile - tile_first) * TILE
|
||||
|
||||
|
||||
def polled_tile(tile_first, tiles, j):
|
||||
"""The pick tile a down round polls as a slot's ``j``-th, the slot's
|
||||
``tiles`` tiles from ``tile_first``: the last past them."""
|
||||
return tile_first + imin(j, tiles - 1)
|
||||
@@ -1,229 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/moe_shape.py
|
||||
"""The MoE stages' build key and route shape: what they are built for (``MoeBuild``),
|
||||
and the task counts, placements and LDS route table that follow from its
|
||||
routing (``RouteShape``), with the table's readers. Plain Python but for the
|
||||
readers; the kernel's stages are in ``moe``."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import flydsl.expr as fx
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import BLOCKS, WAVES
|
||||
from vllm.models.deepseek_v41.amd.mono.stages import moe_plan as mplan
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.dims import UG_PART, Dims
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.gemv import ROWS, TILE, token_tiles
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.router import PARTS as ROUTER_PARTS
|
||||
|
||||
EXPERTS, TOPK = 384, 6 # the target's routing (a build's: ``MoeBuild``)
|
||||
ROUTER0 = 8 # the router's first task: past the xq tasks (a token each)
|
||||
MASK_WORDS = 2 # an expert's token bitmap in the route table: tokens <= 64
|
||||
# a route lane's experts (EXPERTS / 64) sorted descending, ties kept in index
|
||||
# order: the network by their count
|
||||
SORT_NETS = {
|
||||
6: [
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(4, 5),
|
||||
(0, 2),
|
||||
(1, 4),
|
||||
(3, 5),
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(4, 5),
|
||||
(1, 2),
|
||||
(3, 4),
|
||||
(2, 3),
|
||||
],
|
||||
2: [(0, 1)],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoeBuild:
|
||||
tokens: int
|
||||
# routed experts and a token's top-k: the target's, or the DSpark draft's
|
||||
experts: int = EXPERTS
|
||||
topk: int = TOPK
|
||||
# the TP size: every width a rank holds (``Dims``)
|
||||
tp: int = 4
|
||||
timeline: bool = field(default=False, metadata={"sym": "tl"})
|
||||
# FP4 groups an ug task (``RouteShape.ug_groups``)
|
||||
ug_groups: int = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteShape:
|
||||
"""A build's routing: ``experts`` routed experts, ``topk`` a token of
|
||||
``tokens``, the task counts and placements that follow, and the route table
|
||||
``load_route`` lays out in LDS: every token's (ids, weights), |U|, the
|
||||
experts of U, each (t, k)'s slot in U, each expert's slot, the picks in slot
|
||||
order and each slot's first position in it (past one token tile also each
|
||||
slot's first pick tile and each pick tile's slot)."""
|
||||
|
||||
experts: int
|
||||
topk: int
|
||||
tokens: int
|
||||
tp: int
|
||||
# FP4 groups (32 intermediate columns) an ug task, its waves splitting K in
|
||||
# halves (2) or quarters (1). 1 is twice the tasks, so the CTAs' ug ends
|
||||
# closer together: faster at the draft's widths (40 rows -4 us), slower at
|
||||
# the target's (48 rows +3 us; the runners pick)
|
||||
ug_groups: int = 2
|
||||
|
||||
@property
|
||||
def dims(self) -> Dims:
|
||||
return Dims(self.tp)
|
||||
|
||||
@property
|
||||
def ug_parts(self) -> int:
|
||||
"""An expert's ug tasks (``moe_plan.ug_parts``)."""
|
||||
return mplan.ug_parts(self.dims.inter_real // UG_PART, self.ug_groups)
|
||||
|
||||
@property
|
||||
def ug_k_slices(self) -> int:
|
||||
"""The K slices of an ug task's waves: a wave a (group, 16-column half,
|
||||
slice)."""
|
||||
return WAVES // (2 * self.ug_groups)
|
||||
|
||||
@property
|
||||
def picks(self) -> int:
|
||||
return self.tokens * self.topk
|
||||
|
||||
@property
|
||||
def max_u(self) -> int:
|
||||
return min(self.picks, self.experts)
|
||||
|
||||
@property
|
||||
def router_groups(self) -> tuple[int, int]:
|
||||
"""``stage_router``'s token groups and their tokens: a group a token
|
||||
tile, as many as one round of CTAs holds (a CTA running two serialized
|
||||
them: the route waited on the 32 that did at 48 rows)."""
|
||||
per_group = self.experts // ROWS * ROUTER_PARTS
|
||||
groups = max(1, min(len(token_tiles(self.tokens)), BLOCKS // per_group))
|
||||
tokens = -(-self.tokens // groups)
|
||||
assert (groups - 1) * tokens < self.tokens
|
||||
return groups, tokens
|
||||
|
||||
@property
|
||||
def router_tasks(self) -> int:
|
||||
# (token group, 16-row tile, K part): ``stage_router``
|
||||
return self.router_groups[0] * self.experts // ROWS * ROUTER_PARTS
|
||||
|
||||
@property
|
||||
def router0(self) -> int:
|
||||
"""Past the xq tasks (a token each from CTA 0)."""
|
||||
return max(ROUTER0, self.tokens)
|
||||
|
||||
@property
|
||||
def route0(self) -> int:
|
||||
return self.router0 + self.router_tasks
|
||||
|
||||
@property
|
||||
def shared0(self) -> int:
|
||||
return self.route0 + max(8, self.tokens)
|
||||
|
||||
@property
|
||||
def shared_units(self) -> int:
|
||||
"""The shared expert's (task, token tile) units (``stage_shared``)."""
|
||||
return self.dims.shared_tasks * len(token_tiles(self.tokens))
|
||||
|
||||
@property
|
||||
def ug0(self) -> int:
|
||||
"""The first ug round's placement: the shared units' CTAs last."""
|
||||
return self.shared0 + self.shared_units
|
||||
|
||||
@property
|
||||
def sort_net(self):
|
||||
return SORT_NETS[self.experts // 64]
|
||||
|
||||
@property
|
||||
def tab_words(self) -> int:
|
||||
return self.tile_slot0 + self.max_tiles if self.multi else self.queue0 + 1
|
||||
|
||||
@property
|
||||
def multi(self) -> bool:
|
||||
"""More than one token tile: an ug task is a (tile of a slot's picks,
|
||||
part) (``stage_ug``)."""
|
||||
return self.tokens > TILE
|
||||
|
||||
@property
|
||||
def slot_tiles(self) -> int:
|
||||
"""A slot's pick tiles at most (a token picks an expert once)."""
|
||||
return len(token_tiles(self.tokens))
|
||||
|
||||
@property
|
||||
def max_tiles(self) -> int:
|
||||
"""The pick tiles of U at most: a slot's partial tile each, then the
|
||||
whole tiles, and never more than a tile a pick."""
|
||||
return min(self.picks, self.max_u + self.picks // TILE)
|
||||
|
||||
@property
|
||||
def tile_first0(self) -> int:
|
||||
"""Past one token tile: each slot's first pick tile, then their count."""
|
||||
return self.queue0 + 1
|
||||
|
||||
@property
|
||||
def tile_slot0(self) -> int:
|
||||
"""Past one token tile: each pick tile's slot of U."""
|
||||
return self.tile_first0 + self.max_u + 1
|
||||
|
||||
@property
|
||||
def queue0(self) -> int:
|
||||
"""The ug task this CTA took from the queue (``take_ug_task``)."""
|
||||
return self.mask0 + MASK_WORDS * self.experts
|
||||
|
||||
@property
|
||||
def expert_slot0(self) -> int:
|
||||
"""The route table's per-expert slots (a slot where the expert is in U)."""
|
||||
return 3 * self.picks + 1 + self.max_u
|
||||
|
||||
@property
|
||||
def order0(self) -> int:
|
||||
"""The picks (t topk + k) in slot order, a slot's ascending."""
|
||||
return self.expert_slot0 + self.experts
|
||||
|
||||
@property
|
||||
def first0(self) -> int:
|
||||
"""Each slot's first position in the pick order, then the pick count."""
|
||||
return self.order0 + self.picks
|
||||
|
||||
@property
|
||||
def mask0(self) -> int:
|
||||
"""Each expert's tokens that picked it, a bit a token (``MASK_WORDS``)."""
|
||||
return self.first0 + self.max_u + 1
|
||||
|
||||
@property
|
||||
def tile_picks(self) -> int:
|
||||
"""A slot's picks at most: a token picks an expert once."""
|
||||
return min(self.tokens, 16)
|
||||
|
||||
def route_w(self, tab, s, t, k):
|
||||
return fx.ptr_load(tab + (t * 2 * self.topk + self.topk + k)).bitcast(
|
||||
fx.Float32
|
||||
)
|
||||
|
||||
def n_union(self, tab, s):
|
||||
return fx.ptr_load(tab + 2 * s * self.topk)
|
||||
|
||||
def union_expert(self, tab, s, u):
|
||||
return fx.ptr_load(tab + (2 * s * self.topk + 1 + u))
|
||||
|
||||
def slot_first(self, tab, u):
|
||||
return fx.ptr_load(tab + (self.first0 + u))
|
||||
|
||||
def pick_at(self, tab, i):
|
||||
return fx.ptr_load(tab + (self.order0 + i))
|
||||
|
||||
def tile_first(self, tab, u):
|
||||
return fx.ptr_load(tab + (self.tile_first0 + u))
|
||||
|
||||
def tile_slot(self, tab, i):
|
||||
return fx.ptr_load(tab + (self.tile_slot0 + i))
|
||||
|
||||
|
||||
def route_shape(key: MoeBuild) -> RouteShape:
|
||||
return RouteShape(key.experts, key.topk, key.tokens, key.tp, key.ug_groups)
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/model_ops/deepseek_v41/router.py
|
||||
"""V4.1's router GEMV tile for decode: bf16 x [T, 5120] times the gate weight
|
||||
[384, 5120] -> f32 logits, in one fixed summation order.
|
||||
|
||||
Rows 16 a tile, K in ``PARTS`` contiguous parts, 16x16x32 bf16 MFMA: in part
|
||||
p wave w takes the part's K steps w, w + 8, ... (32 columns each) in that
|
||||
order, and the eight wave partials are summed in wave order
|
||||
(``common.ops.row_sum``); the MoE's route stage sums the parts in part
|
||||
order. The MoE stages spread a tile's parts over PARTS CTAs, and past 16 tokens
|
||||
a token tile a CTA too, which the order allows: a CTA alone cannot read a
|
||||
tile, or every token's x, fast enough.
|
||||
"""
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import mfma_bf16, rsrc
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import WAVES
|
||||
|
||||
HIDDEN = 5120
|
||||
ROWS = 16 # a tile's rows: the MFMA's M
|
||||
PARTS = 4 # contiguous K parts of a tile, summed in order
|
||||
|
||||
|
||||
def _k0s(part, lane, wave):
|
||||
"""The K offsets of this lane's loads in part ``part`` (8 bf16 each)."""
|
||||
per_wave = HIDDEN // 32 // WAVES // PARTS
|
||||
assert per_wave * 32 * WAVES * PARTS == HIDDEN
|
||||
return [
|
||||
(part * per_wave * WAVES + wave + WAVES * i) * 32 + 8 * (lane // 16)
|
||||
for i in range(per_wave)
|
||||
]
|
||||
|
||||
|
||||
def router_weight_loads(w, task, part, lane, wave, w_cm=0):
|
||||
"""``router_tile``'s weight operands (rows 16 task .., K part ``part``).
|
||||
``w_cm``: their load cache policy (nontemporal where nothing else of the
|
||||
step reuses them)."""
|
||||
row = task * ROWS + lane % 16
|
||||
return [
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(w),
|
||||
(row * HIDDEN + k0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=w_cm,
|
||||
)
|
||||
)
|
||||
for k0 in _k0s(part, lane, wave)
|
||||
]
|
||||
|
||||
|
||||
def router_x_loads(x, part, s, lane, wave, x_cm=0, t0=0):
|
||||
"""``router_tile``'s x operands: the s tokens from ``t0`` (column
|
||||
lane % 16; ``t0`` traced or an int). ``x_cm``: their load cache policy
|
||||
(device scope when another CTA of the same launch wrote x)."""
|
||||
tok = fx.min(lane % 16, s - 1)
|
||||
if const_expr(not isinstance(t0, int) or t0 != 0):
|
||||
tok = tok + t0
|
||||
return [
|
||||
fx.Vector(
|
||||
bo.buffer_load(
|
||||
rsrc(x),
|
||||
(tok * HIDDEN + k0) // 2,
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=x_cm,
|
||||
)
|
||||
)
|
||||
for k0 in _k0s(part, lane, wave)
|
||||
]
|
||||
|
||||
|
||||
def router_tile_mfma(wvs, xvs, lane, wave, red):
|
||||
"""``router_tile``'s MFMAs, K steps in order -> ``red[wave][lane]``."""
|
||||
acc = fx.Vector.filled(4, 0.0, fx.Float32)
|
||||
for i in range_constexpr(len(wvs)):
|
||||
acc = mfma_bf16(wvs[i].bitcast(fx.BFloat16), xvs[i].bitcast(fx.BFloat16), acc)
|
||||
fx.ptr_store(acc, red + (wave * 64 + lane) * 4)
|
||||
@@ -1,412 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_pre.py (slice, gate, norm) and
|
||||
# atom/models/deepseek_v41/mono/kernels/attn_post.py (the attention's TP reduce).
|
||||
"""The mono layer's mHC seam stages, each a task table of a persistent launch
|
||||
handing off through the tagged mailbox:
|
||||
|
||||
slice (160, 32 hidden columns x 4 streams each): the owed post folded into
|
||||
the residual (R_new, bf16 out), the collapsed layer input (mailbox
|
||||
LIN) and the partial mix projection / sum of squares (mailbox PMIX)
|
||||
gate (S): the 160 partials -> post / comb (Sinkhorn) and the next pre
|
||||
norm (S): the RMSNorm -> the bf16 row, handed to the caller's store
|
||||
reduce (160): the FFN seam's attention output, the TP ranks' partials
|
||||
(pushed to every rank's ATTN region) summed in the all-reduce's
|
||||
order -> ``pend_lds``, where its slice reads it
|
||||
push (160): a rank's partial given in memory (an attention vLLM ran)
|
||||
to every rank's ATTN region, ahead of the slice's reduce
|
||||
|
||||
Every elementwise step follows aiter's ``mhc_fused_post_pre_delayed_rmsnorm``
|
||||
rounding; the mix projection and the sums of squares, split-K reductions,
|
||||
differ only in their split and order.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
||||
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
||||
from flydsl.expr import math as fmath
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
||||
batched_rounds,
|
||||
bf16_round,
|
||||
bf_hi,
|
||||
bf_lo,
|
||||
butterfly,
|
||||
hw_exp2,
|
||||
hw_rcp,
|
||||
hw_rsq,
|
||||
lane_gather,
|
||||
rsrc,
|
||||
sum_partials,
|
||||
traced,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.common.plan import THREADS, WAVES, pair_layout
|
||||
from vllm.models.deepseek_v41.amd.mono.common.sync import preg
|
||||
from vllm.models.deepseek_v41.amd.mono.stages.gemv import (
|
||||
ld_bf,
|
||||
ld_f32,
|
||||
plus,
|
||||
token_tiles,
|
||||
)
|
||||
|
||||
HC = 4
|
||||
HIDDEN = 5120
|
||||
K_MIX = HC * HIDDEN
|
||||
MIX = HC * (HC + 2) # 24 projections: pre 4 | post 4 | comb 16
|
||||
EPS = 1e-20
|
||||
HC_EPS = 1e-6
|
||||
SINKHORN = 20
|
||||
POST_MULT = 2.0
|
||||
LOG2E = 1.0 / math.log(2.0)
|
||||
|
||||
COLS = 32 # hidden columns a slice task
|
||||
SLICES = HIDDEN // COLS # 160
|
||||
KT = HC * COLS # projection columns a slice task
|
||||
PART = MIX + 1 # a slice's partials a token: the mixes, then the sum of squares
|
||||
GATE_BLOCK = 8 # slices a gate thread sums
|
||||
|
||||
|
||||
def scratch_layout(tokens: int) -> dict[str, tuple[int, int]]:
|
||||
"""Mailbox region -> (byte offset, bytes) of a seam; 8 B a pair."""
|
||||
pairs = {
|
||||
"lin": tokens * HIDDEN // 2, # bf16 pairs
|
||||
"pmix": SLICES * tokens * PART,
|
||||
}
|
||||
return pair_layout(pairs.items())
|
||||
|
||||
|
||||
def attn_peer_bytes(tokens: int, tp: int) -> int:
|
||||
"""ATTN: ``[source rank][token][hidden / 2]`` bf16 pairs."""
|
||||
return tp * tokens * HIDDEN // 2 * 8
|
||||
|
||||
|
||||
def attn_region_pair(src, t, s, col):
|
||||
return (src * s + t) * (HIDDEN // 2) + col // 2
|
||||
|
||||
|
||||
def fma(a, b, c):
|
||||
return fx.Float32(fmath.fma(a, b, c))
|
||||
|
||||
|
||||
def sigmoid(z):
|
||||
"""Triton's ``tl.sigmoid``: 1 / (1 + exp2(-z log2(e)))."""
|
||||
return fx.Float32(1.0) / (fx.Float32(1.0) + hw_exp2(z * fx.Float32(-LOG2E)))
|
||||
|
||||
|
||||
def lane_f32(v, src):
|
||||
return lane_gather(v.bitcast(fx.Int32), src).bitcast(fx.Float32)
|
||||
|
||||
|
||||
def pend_value(c, t, col, cc):
|
||||
"""The owed post's sublayer output at (token t, column col): read from
|
||||
``pend``, or where a caller staged it (``pend_lds``, ``[token][32]`` of this
|
||||
task's columns: a reduction it did first)."""
|
||||
if const_expr(c.get("pend_lds") is None):
|
||||
return ld_bf(c["args"]["pend"], t * HIDDEN + col)
|
||||
return fx.ptr_load(c["pend_lds"] + (t * COLS + cc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- slice
|
||||
@traced
|
||||
def slice_input(c, task, idx):
|
||||
"""Thread ``idx``'s pair of the layer input (token idx / 16, columns
|
||||
32 task + 2 (idx % 16) ..)."""
|
||||
s, rl = c["S"], c["rl"]
|
||||
a = c["args"]
|
||||
c0 = task * COLS
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
cc = 2 * (idx % (COLS // 2))
|
||||
outs = []
|
||||
for d in range_constexpr(2):
|
||||
acc = fx.Float32(0.0)
|
||||
for h in range_constexpr(HC):
|
||||
rb = fx.ptr_load(rl + (t * KT + h * COLS + cc + d))
|
||||
acc = acc + ld_f32(a["pre_in"], t * HC + h) * rb
|
||||
outs.append(acc)
|
||||
c["put_bf"](c["lin"], t * HIDDEN + c0 + cc, outs)
|
||||
|
||||
|
||||
def slice_at(c0, e):
|
||||
"""Element e of a slice task at column c0: (token, stream, hidden column)."""
|
||||
return e // KT, (e % KT) // COLS, c0 + e % COLS
|
||||
|
||||
|
||||
def slice_loads(c, c0, e):
|
||||
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
if const_expr(not c.get("fold", True)):
|
||||
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
|
||||
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
|
||||
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
|
||||
return r, cm, ld_f32(a["post_in"], t * HC + o)
|
||||
|
||||
|
||||
def slice_row(c, c0, e, got):
|
||||
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
if const_expr(not c.get("fold", True)):
|
||||
# a settled residual (no owed post): stream o as it arrives
|
||||
v = got[0]
|
||||
else:
|
||||
r, cm, post = got
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
|
||||
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
|
||||
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
|
||||
for h in range_constexpr(1, HC - 1):
|
||||
v = v + cm[h] * r[h]
|
||||
# bf16 R_new is what the collapse, the sums of squares and the
|
||||
# mix projection all read
|
||||
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
|
||||
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
|
||||
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
|
||||
|
||||
|
||||
@traced
|
||||
def stage_slice(c, task):
|
||||
"""R_new, the layer input and the partial projections of hidden columns
|
||||
32 task .. +32, all four streams, every token."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
rl, fl = c["rl"], c["fl"]
|
||||
a = c["args"]
|
||||
c0 = task * COLS
|
||||
batched_rounds(
|
||||
tid,
|
||||
s * KT,
|
||||
lambda e: slice_loads(c, c0, e),
|
||||
lambda e, got: slice_row(c, c0, e, got),
|
||||
)
|
||||
for i in range_constexpr((MIX * KT + THREADS - 1) // THREADS):
|
||||
e = tid + THREADS * i
|
||||
if e < MIX * KT:
|
||||
j = e // KT
|
||||
o = (e % KT) // COLS
|
||||
# the original's two bf16 MFMAs take fn as hi = bf16(fn) and lo =
|
||||
# bf16(fn - hi): their sum, exact in fp32, is the weight it applies
|
||||
f = ld_f32(a["hc_fn"], j * K_MIX + o * HIDDEN + c0 + e % COLS)
|
||||
hi = bf16_round(f)
|
||||
fx.ptr_store(hi + bf16_round(f - hi), fl + e)
|
||||
gpu.barrier()
|
||||
# the layer input: bf16 R_new of the four streams by the incoming pre-mix,
|
||||
# products rounded, summed in order (the Triton collapse, no contraction);
|
||||
# two columns a thread, one bf16 pair a mailbox store
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
slice_input(c, task, plus(pas, tid))
|
||||
# sum of squares: token t on wave t % WAVES, two columns a lane
|
||||
for t in range_constexpr(s):
|
||||
if wave == t % WAVES:
|
||||
x0 = fx.ptr_load(rl + (t * KT + 2 * lane))
|
||||
x1 = fx.ptr_load(rl + (t * KT + 2 * lane + 1))
|
||||
sq = butterfly(x0 * x0 + x1 * x1, (32, 16, 8, 4, 2, 1))
|
||||
if lane == 0:
|
||||
c["put"](c["pmix"], (task * s + t) * PART + MIX, sq)
|
||||
# mixes: fp32 MFMA 16x16x4, A = fn rows (two 16-row tiles), B = R_new^T,
|
||||
# a token tile at a time
|
||||
for t0, n in token_tiles(s):
|
||||
slice_mix(c, task, t0, n)
|
||||
|
||||
|
||||
@traced
|
||||
def slice_mix(c, task, t0, n):
|
||||
"""The mix projection partials of tokens t0 .. t0 + n (``stage_slice``)."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
rl, fl, red = c["rl"], c["fl"], c["red"]
|
||||
tok = plus(t0, fx.min(lane % 16, n - 1))
|
||||
kw = KT // WAVES
|
||||
cs = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(2)]
|
||||
for kk in range_constexpr(kw // 4):
|
||||
colk = wave * kw + lane // 16 + 4 * kk
|
||||
b = fx.ptr_load(rl + (tok * KT + colk))
|
||||
for h in range_constexpr(2):
|
||||
row = h * 16 + lane % 16
|
||||
av = fx.ptr_load(fl + (fx.min(row, MIX - 1) * KT + colk))
|
||||
av = (row < MIX).select(av, fx.Float32(0.0))
|
||||
ops = rocdl._split_mfma_operands([av, b, cs[h], 0, 0, 0])
|
||||
cs[h] = fx.Vector(rocdl.mfma_f32_16x16x4f32(T.vec(4, T.f32), *ops).result)
|
||||
for h in range_constexpr(2):
|
||||
fx.ptr_store(cs[h], red + ((wave * 2 + h) * 64 + lane) * 4)
|
||||
gpu.barrier()
|
||||
if tid < MIX * n:
|
||||
tl = tid // MIX
|
||||
t = plus(t0, tl)
|
||||
j = tid % MIX
|
||||
h = j // 16
|
||||
ln = 16 * ((j % 16) // 4) + tl
|
||||
tot = fx.Float32(0.0)
|
||||
for w in range_constexpr(WAVES):
|
||||
tot = tot + fx.ptr_load(red + (((w * 2 + h) * 64 + ln) * 4 + j % 4))
|
||||
c["put"](c["pmix"], (task * s + t) * PART + j, tot)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- gate
|
||||
def _row(v, lane, q):
|
||||
"""Comb lanes 8..23 hold ``comb[r][c]`` at lane 8 + 4 r + c: element q of
|
||||
the row."""
|
||||
return lane_f32(v, (lane - 8) // 4 * 4 + 8 + q)
|
||||
|
||||
|
||||
def _col(v, lane, q):
|
||||
return lane_f32(v, 8 + 4 * q + (lane - 8) % 4)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_gate(c, t):
|
||||
"""Token t's 160 slices, in slice order, -> post / comb / next pre."""
|
||||
s, tid, lane, wave = c["S"], c["tid"], c["lane"], c["wave"]
|
||||
red = c["red"]
|
||||
a = c["args"]
|
||||
groups = SLICES // GATE_BLOCK # 20
|
||||
if tid < groups * PART:
|
||||
g = tid // PART
|
||||
j = tid % PART
|
||||
vals = c["poll"](
|
||||
[
|
||||
(c["pmix"], ((g * GATE_BLOCK + k) * s + t) * PART + j, 1)
|
||||
for k in range(GATE_BLOCK)
|
||||
]
|
||||
)
|
||||
acc = vals[0][0].bitcast(fx.Float32)
|
||||
for k in range_constexpr(1, GATE_BLOCK):
|
||||
acc = acc + vals[k][0].bitcast(fx.Float32)
|
||||
fx.ptr_store(acc, red + (g * PART + j))
|
||||
gpu.barrier()
|
||||
if wave == 0:
|
||||
j = fx.min(lane, MIX - 1)
|
||||
mix = fx.ptr_load(red + j)
|
||||
sq = fx.ptr_load(red + MIX)
|
||||
for g in range_constexpr(1, groups):
|
||||
mix = mix + fx.ptr_load(red + (g * PART + j))
|
||||
sq = sq + fx.ptr_load(red + (g * PART + MIX))
|
||||
sc = [ld_f32(a["hc_scale"], i) for i in range(3)]
|
||||
base = ld_f32(a["hc_base"], j)
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm reduce: one rstd =
|
||||
# rsq(fma(sum, 1/K, eps)), then each gate fma(mix rstd, scale, base)
|
||||
v = mix * hw_rsq(fma(sq, fx.Float32(1.0 / K_MIX), fx.Float32(EPS)))
|
||||
if lane < HC:
|
||||
pre = sigmoid(fma(v, sc[0], base))
|
||||
bo.buffer_store(pre + fx.Float32(HC_EPS), rsrc(a["pre_out"]), t * HC + lane)
|
||||
if (lane >= HC) & (lane < 2 * HC):
|
||||
post = sigmoid(fma(v, sc[1], base))
|
||||
bo.buffer_store(
|
||||
post * fx.Float32(POST_MULT), rsrc(a["post_out"]), t * HC + lane - HC
|
||||
)
|
||||
cv = fma(v, sc[2], base)
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
# Triton's exp: exp2 of (x - max) log2(e)
|
||||
m = fx.max(fx.max(row[0], row[1]), fx.max(row[2], row[3]))
|
||||
cv = hw_exp2((cv - m) * fx.Float32(LOG2E))
|
||||
# fast_dividef is x rcp(y); its first "+ eps" contracts into the fma
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
cv = fma(cv, hw_rcp((row[0] + row[1]) + (row[2] + row[3])), fx.Float32(HC_EPS))
|
||||
col = [_col(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS))
|
||||
for _ in range_constexpr(SINKHORN - 1):
|
||||
row = [_row(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(
|
||||
((row[0] + row[1]) + (row[2] + row[3])) + fx.Float32(HC_EPS)
|
||||
)
|
||||
col = [_col(cv, lane, q) for q in range(4)]
|
||||
cv = cv * hw_rcp(
|
||||
((col[0] + col[1]) + (col[2] + col[3])) + fx.Float32(HC_EPS)
|
||||
)
|
||||
if (lane >= 2 * HC) & (lane < MIX):
|
||||
bo.buffer_store(cv, rsrc(a["comb_out"]), t * HC * HC + lane - 2 * HC)
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- norm
|
||||
@traced
|
||||
def stage_norm(c, t, store):
|
||||
"""Token t's RMSNorm (rstd = rsq(fma(sum, 1/H, eps)), bf16((x rstd) w));
|
||||
256 threads, 3 chunks of 8 at 8 t + 2048 c, each chunk's bf16 values handed
|
||||
to ``store(t, col, ys, live)`` by every thread (``live``: a thread's chunk
|
||||
in the row; threads 256.. mirror thread 255)."""
|
||||
tid, lane, wave, red = c["tid"], c["lane"], c["wave"], c["red"]
|
||||
a = c["args"]
|
||||
tt = fx.min(tid, 255)
|
||||
mine = tid < 256
|
||||
cols = [tt * 8 + 2048 * ch for ch in range(3)]
|
||||
xs = []
|
||||
for ch in range_constexpr(3):
|
||||
base = t * HIDDEN + fx.min(cols[ch], HIDDEN - 8)
|
||||
words = c["poll"]([(c["lin"], base // 2 + k, 1) for k in range(4)])
|
||||
v = []
|
||||
for k in range_constexpr(4):
|
||||
v += [bf_lo(words[k][0]), bf_hi(words[k][0])]
|
||||
xs.append([(cols[ch] < HIDDEN).select(x, fx.Float32(0.0)) for x in v])
|
||||
acc = fx.Float32(0.0)
|
||||
for ch in range_constexpr(3):
|
||||
for x in xs[ch]:
|
||||
acc = acc + x * x
|
||||
acc = butterfly(acc, (1, 2, 4, 8, 16, 32))
|
||||
if (lane == 0) & mine:
|
||||
fx.ptr_store(acc, red + wave)
|
||||
gpu.barrier()
|
||||
tot = butterfly(fx.ptr_load(red + lane % 4), (1, 2))
|
||||
r = hw_rsq(fma(tot, fx.Float32(1.0 / HIDDEN), fx.Float32(EPS)))
|
||||
for ch in range_constexpr(3):
|
||||
col = fx.min(cols[ch], HIDDEN - 8)
|
||||
ys = [
|
||||
bf16_round((xs[ch][j] * r) * ld_bf(a["norm_w"], col + j)) for j in range(8)
|
||||
]
|
||||
store(t, col, ys, mine & (cols[ch] < HIDDEN))
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reduce
|
||||
@traced
|
||||
def stage_reduce(c, task):
|
||||
"""The attention output at this slice's 32 columns: the TP ranks' partials
|
||||
in the all-reduce's order (``sum_partials``), in fp32 -> bf16 ->
|
||||
``pend_lds``."""
|
||||
s, tid = c["S"], c["tid"]
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
reduce_attn_pair(c, task, plus(pas, tid))
|
||||
gpu.barrier()
|
||||
|
||||
|
||||
@traced
|
||||
def reduce_attn_pair(c, task, idx):
|
||||
"""Thread ``idx``'s column pair of token idx / 16 (``stage_reduce``)."""
|
||||
s = c["S"]
|
||||
c0 = task * COLS
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
col = c0 + 2 * (idx % (COLS // 2))
|
||||
own = preg(c["sym"], 0, "attn")
|
||||
acc0, acc1 = sum_partials(
|
||||
c["poll"],
|
||||
own,
|
||||
lambda src: attn_region_pair(src, t, s, col),
|
||||
c["d"].tp,
|
||||
)
|
||||
lds = c["pend_lds"] + (t * COLS + col - c0)
|
||||
fx.ptr_store(bf16_round(acc0), lds)
|
||||
fx.ptr_store(bf16_round(acc1), lds + 1)
|
||||
|
||||
|
||||
@traced
|
||||
def stage_push(c, task, part):
|
||||
"""This rank's attention partial (``part``: bf16 [S, HIDDEN], wo_b's
|
||||
unreduced output) at this slice's 32 columns, every token, to every rank's
|
||||
ATTN region: the pairs ``stage_reduce`` polls, as the bits vLLM wrote."""
|
||||
s, tid, tp = c["S"], c["tid"], c["d"].tp
|
||||
for pas in range_constexpr(0, s * COLS // 2, THREADS):
|
||||
idx = plus(pas, tid)
|
||||
if idx < s * COLS // 2:
|
||||
t = idx // (COLS // 2)
|
||||
col = task * COLS + 2 * (idx % (COLS // 2))
|
||||
w = bo.buffer_load(
|
||||
rsrc(part), (t * HIDDEN + col) // 2, vec_width=1, dtype=T.i32
|
||||
)
|
||||
pair = attn_region_pair(c["rank"], t, s, col)
|
||||
for p in range_constexpr(tp):
|
||||
c["put_words"](preg(c["peer_addr"](p), 0, "attn"), pair, [w])
|
||||
@@ -1,435 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
|
||||
|
||||
A decode step's backbone layer -- the attention seam, the attention, its TP
|
||||
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
|
||||
FlyDSL launches (``mono``, this directory) on the layer's loaded
|
||||
weights, the TP reductions inside the kernels. Every backbone layer of the
|
||||
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
|
||||
the first layer and the Engram layers hand K1 a settled residual, and an index
|
||||
source runs vLLM's compressor and indexer between the two launches. A layer
|
||||
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
|
||||
and attention and runs the rest as one launch, the FFN launch, from wo_b's
|
||||
unreduced output. A step takes the kernels when it is decode only, at most
|
||||
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
|
||||
a whole layer also needs causal SWA windows. Anything else runs the layer as
|
||||
before; the paths share every tensor at the layer boundary, so they interleave
|
||||
freely.
|
||||
|
||||
Every TP rank must take the same path at every layer (the kernels wait on each
|
||||
other's pushes): the conditions read only metadata every rank holds alike.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterSparseSWAMetadata
|
||||
from vllm.models.deepseek_v41.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RECORD = 584 # an fp8_ds_mla KV record: 576 data bytes, 8 scale bytes
|
||||
SWA_WIDTH = 128 # a causal decode token's window slots
|
||||
MAX_ROWS = 48 # the kernels' step rows at most: 8 requests x (1 + 5 DSpark drafts)
|
||||
# the kernels' ISA: CDNA4 (gfx950) scaled MFMA and MX formats
|
||||
CDNA_VERSIONS = (4,)
|
||||
|
||||
_runner = None
|
||||
|
||||
|
||||
def _mono_runner(device: torch.device):
|
||||
"""The process's mono runner (one a TP rank, shared by every layer). Its
|
||||
peer-memory handshake is collective over the TP group: every rank builds it
|
||||
at the same layer of the same step."""
|
||||
global _runner
|
||||
if _runner is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
# its peer memory is allocated and exchanged eagerly; vLLM warms
|
||||
# every graph up eagerly before capturing it
|
||||
raise RuntimeError(
|
||||
"--monokernel was first reached inside a CUDA graph "
|
||||
"capture, before an eager decode step."
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
|
||||
|
||||
assert MAX_TOKENS >= MAX_ROWS, MAX_TOKENS
|
||||
_runner = DSV41MonoLayer(
|
||||
get_tensor_model_parallel_world_size(),
|
||||
get_tensor_model_parallel_rank(),
|
||||
get_tp_group().cpu_group,
|
||||
device,
|
||||
)
|
||||
return _runner
|
||||
|
||||
|
||||
def _record_view(kv: torch.Tensor, block: int) -> torch.Tensor:
|
||||
"""A layer's fp8_ds_mla cache as [blocks, block, RECORD] bytes (a block's
|
||||
data rows, then its scale words: the kernels index both from it)."""
|
||||
kv = kv if kv.dtype == torch.uint8 else kv.view(torch.uint8)
|
||||
assert kv.stride(0) >= block * RECORD, (tuple(kv.shape), kv.stride(), block)
|
||||
return torch.as_strided(kv, (kv.shape[0], block, RECORD), (kv.stride(0), RECORD, 1))
|
||||
|
||||
|
||||
def _scale_bytes(scale: torch.Tensor) -> torch.Tensor:
|
||||
return scale if scale.dtype == torch.uint8 else scale.view(torch.uint8)
|
||||
|
||||
|
||||
def _compute_units() -> int:
|
||||
return current_platform.num_compute_units(torch.accelerator.current_device_index())
|
||||
|
||||
|
||||
def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None:
|
||||
"""Why the kernels cannot run this deployment's MoE, or None. They read every
|
||||
one of the 384 experts on each rank, TP-sharded, in AITER's A8W4 MXFP4
|
||||
layout."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
|
||||
pc = vllm_config.parallel_config
|
||||
if pc.enable_expert_parallel or pc.enable_eplb or pc.data_parallel_size > 1:
|
||||
return "needs tensor parallelism only: no expert or data parallelism"
|
||||
if envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
|
||||
return "needs AITER's A8W4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 unset)"
|
||||
experts = getattr(getattr(layer.ffn, "experts", None), "routed_experts", None)
|
||||
if experts is None:
|
||||
return None # not a routed MoE: the layer checks decline it
|
||||
backend = getattr(experts.quant_method, "mxfp4_backend", None)
|
||||
if backend != Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
return f"needs the AITER_MXFP4_BF16 MoE backend, not {backend}"
|
||||
return None
|
||||
|
||||
|
||||
def _select(
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
positions: torch.Tensor,
|
||||
normed: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
) -> None:
|
||||
"""An index source's compressor and top-k selection, from K1's normed input
|
||||
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
|
||||
indexer op, serial."""
|
||||
attn = layer.attn
|
||||
comp, indexer = attn.compressor, attn.indexer
|
||||
assert indexer is not None
|
||||
if comp is not None:
|
||||
kv_score = torch.mm(
|
||||
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
|
||||
)
|
||||
latent = comp(kv_score, positions)
|
||||
comp.insert_cache(latent, positions, attn.rotary_emb)
|
||||
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
|
||||
weights, _ = indexer.weights_proj(normed)
|
||||
q, q_scale, weights = indexer.forward_q(
|
||||
qr, None, weights, positions, attn.indexer_rotary_emb
|
||||
)
|
||||
assert weights is not None
|
||||
indexer.indexer_op(
|
||||
normed, (q, q_scale) if q_scale is not None else q, None, weights
|
||||
)
|
||||
|
||||
|
||||
class MonoDecodeLayer:
|
||||
"""One decoder layer's mono path (``create``: None when the layer is not
|
||||
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
|
||||
launch after vLLM's attention (``ffn``). Its weights are taken from the
|
||||
layer at the first call, after loading."""
|
||||
|
||||
def __init__(self, ffn_only: bool = False) -> None:
|
||||
self.ffn_only = ffn_only
|
||||
self._weights: MonoLayerWeights | None = None
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
layer: "DeepseekV4DecoderLayer", vllm_config
|
||||
) -> "MonoDecodeLayer | None":
|
||||
if not vllm_config.kernel_config.monokernel:
|
||||
return None
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
# the deployment: an explicit opt-in that cannot run is an error
|
||||
why = None
|
||||
if get_cdna_version() not in CDNA_VERSIONS:
|
||||
why = f"needs CDNA{'/'.join(map(str, CDNA_VERSIONS))}"
|
||||
elif get_tensor_model_parallel_world_size() not in (2, 4):
|
||||
why = "needs tensor parallel size 2 or 4"
|
||||
else:
|
||||
why = _unsupported_moe(layer, vllm_config)
|
||||
if why is None:
|
||||
try:
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import BLOCKS
|
||||
except ImportError as err:
|
||||
why = f"needs FlyDSL and AITER's FlyDSL helpers ({err})"
|
||||
else:
|
||||
# every CTA of a launch stays resident: a partitioned GPU deadlocks
|
||||
cus = _compute_units()
|
||||
if cus < BLOCKS:
|
||||
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
|
||||
if why is not None:
|
||||
raise ValueError(f"--monokernel {why}.")
|
||||
# the layer: the kernels serve the backbone's seams and MoE
|
||||
config = vllm_config.model_config.hf_config
|
||||
attn, ffn = layer.attn, layer.ffn
|
||||
if layer.use_sequence_parallel:
|
||||
why = "sequence-parallel layer"
|
||||
elif not layer.fuse_seam_norm:
|
||||
why = "needs aiter's fused mHC seam"
|
||||
elif attn.layer_id >= config.num_hidden_layers:
|
||||
why = "draft layer"
|
||||
elif (
|
||||
ffn.shared_experts is None
|
||||
or ffn.gate.tid2eid is not None
|
||||
or ffn.n_routed_experts != 384
|
||||
or ffn.n_activated_experts != 6
|
||||
or ffn.routed_scaling_factor != 1.5
|
||||
or ffn.swiglu_limit != 10.0
|
||||
or ffn.scoring_func != "sqrtsoftplus"
|
||||
or not ffn.renormalize
|
||||
):
|
||||
why = "MoE routing outside the kernels' (384 / 6, sqrtsoftplus, x1.5)"
|
||||
if why is not None:
|
||||
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
|
||||
return None
|
||||
# the whole layer: the kernels' attention takes the standard layers; an
|
||||
# index source's compressor and indexer run between its two launches
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
if attn.compressor is not None and attn.indexer is None:
|
||||
why = "a compressor without an indexer"
|
||||
elif attn.indexer is not None and not isinstance(
|
||||
attn.indexer, DeepseekV41RocmMxfp4Indexer
|
||||
):
|
||||
why = f"a {type(attn.indexer).__name__}"
|
||||
elif attn.compress_ratio not in (0, 1, 2):
|
||||
why = f"compress ratio {attn.compress_ratio}"
|
||||
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
|
||||
why = f"a {attn.kv_cache_dtype} KV cache"
|
||||
if why is not None:
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode: the FFN launch for layers with %s", why
|
||||
)
|
||||
return MonoDecodeLayer(ffn_only=True)
|
||||
logger.info_once("DSv4.1 mono decode: the whole standard backbone layers")
|
||||
return MonoDecodeLayer()
|
||||
|
||||
def weights(self, layer: "DeepseekV4DecoderLayer"):
|
||||
if self._weights is not None:
|
||||
return self._weights
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MonoLayerWeights
|
||||
|
||||
a, f = layer.attn, layer.ffn
|
||||
e, sh = f.experts.routed_experts, f.shared_experts
|
||||
assert sh is not None # ``create`` took only layers with one
|
||||
self._weights = MonoLayerWeights(
|
||||
attn=None if self.ffn_only else self._attn_weights(a),
|
||||
hc_attn_fn=layer.hc_attn_fn,
|
||||
hc_attn_scale=layer.hc_attn_scale,
|
||||
hc_attn_base=layer.hc_attn_base,
|
||||
attn_norm=layer.attn_norm.weight,
|
||||
hc_ffn_fn=layer.hc_ffn_fn,
|
||||
hc_ffn_scale=layer.hc_ffn_scale,
|
||||
hc_ffn_base=layer.hc_ffn_base,
|
||||
ffn_norm=layer.ffn_norm.weight,
|
||||
gate_w=f.gate.weight,
|
||||
bias=f.gate.e_score_correction_bias,
|
||||
w13=e.w13_weight,
|
||||
w13_s=e.w13_weight_scale,
|
||||
w2=e.w2_weight,
|
||||
w2_s=e.w2_weight_scale,
|
||||
sgu=sh.gate_up_proj.weight,
|
||||
sgu_s=_scale_bytes(sh.gate_up_proj.weight_scale),
|
||||
sw2=sh.down_proj.weight,
|
||||
sw2_s=_scale_bytes(sh.down_proj.weight_scale),
|
||||
)
|
||||
self._weights.check(get_tensor_model_parallel_world_size())
|
||||
return self._weights
|
||||
|
||||
@staticmethod
|
||||
def _attn_weights(a):
|
||||
from vllm.models.deepseek_v41.amd.mono.attention.plan import Dims
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import AttnWeights
|
||||
|
||||
attn = AttnWeights(
|
||||
layer_id=a.layer_id,
|
||||
wqkv=a.fused_wqa_wkv.weight,
|
||||
wqkv_scale=_scale_bytes(a.fused_wqa_wkv.weight_scale),
|
||||
q_norm=a.q_norm.weight,
|
||||
kv_norm=a.kv_norm.weight,
|
||||
wq_b=a.wq_b.weight,
|
||||
wq_b_scale=_scale_bytes(a.wq_b.weight_scale),
|
||||
wo_a=a.wo_a.weight,
|
||||
wo_a_scale=_scale_bytes(a.wo_a.weight_scale),
|
||||
wo_b=a.wo_b.weight,
|
||||
wo_b_scale=_scale_bytes(a.wo_b.weight_scale),
|
||||
attn_sink=a.attn_sink,
|
||||
cos_sin=a.rotary_emb.cos_sin_cache,
|
||||
ratio=a.compress_ratio,
|
||||
)
|
||||
attn.check(Dims(get_tensor_model_parallel_world_size()))
|
||||
return attn
|
||||
|
||||
@staticmethod
|
||||
def _skip(why: str) -> None:
|
||||
logger.debug_once("DSv4.1 mono decode skipped a step: %s", why)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _decode_step(layer: "DeepseekV4DecoderLayer", rows: int):
|
||||
"""(why this step takes the original path -- None for a decode-only step
|
||||
the kernels take -- the layer's SWA metadata, its compressed cache's)."""
|
||||
if not 1 <= rows <= MAX_ROWS:
|
||||
return f"{rows} rows", None, None
|
||||
if not is_forward_context_available():
|
||||
return "no forward context", None, None
|
||||
fc = get_forward_context()
|
||||
md = fc.attn_metadata
|
||||
attn = layer.attn
|
||||
swa, comp = None, None
|
||||
if isinstance(md, dict):
|
||||
swa = cast(
|
||||
"DeepseekV41ROCMAiterSparseSWAMetadata | None",
|
||||
md.get(attn.swa_cache_layer.prefix),
|
||||
)
|
||||
if attn.compressed_cache_prefix is not None:
|
||||
comp = cast(
|
||||
"DeepseekV4FlashMLAMetadata | None",
|
||||
md.get(attn.compressed_cache_prefix),
|
||||
)
|
||||
why = None
|
||||
if fc.cudagraph_runtime_mode not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL):
|
||||
# a PIECEWISE (breakable) capture is replayed for mixed batches too
|
||||
why = f"{fc.cudagraph_runtime_mode} step"
|
||||
elif not isinstance(md, dict): # a profile / dummy run, or DBO ubatches
|
||||
why = "no attention metadata"
|
||||
elif swa is None:
|
||||
why = "layer metadata missing"
|
||||
elif swa.num_prefills != 0 or swa.num_decodes == 0:
|
||||
why = "not a decode-only step"
|
||||
elif swa.num_decode_tokens > rows:
|
||||
why = "more decode tokens than rows"
|
||||
return why, swa, comp
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor | None,
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, ...] | None:
|
||||
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
|
||||
mono kernels, or None when this step takes the original path (always for
|
||||
an ``ffn_only`` layer)."""
|
||||
if self.ffn_only:
|
||||
return None
|
||||
first = residual is None
|
||||
if first and x.dim() != 2:
|
||||
return self._skip("first layer of a pipeline stage")
|
||||
why, swa, comp = self._decode_step(layer, x.shape[0])
|
||||
attn = layer.attn
|
||||
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
|
||||
if why is None:
|
||||
# the kernels' attention: causal SWA windows, the compressed cache
|
||||
if comp is None and attn.compress_ratio:
|
||||
why = "layer metadata missing"
|
||||
elif swa.decode_swa_width != SWA_WIDTH:
|
||||
why = f"SWA width {swa.decode_swa_width}"
|
||||
elif (
|
||||
swa.decode_swa_indices is None
|
||||
or swa.decode_swa_lens is None
|
||||
or swa.token_to_req_indices is None
|
||||
):
|
||||
why = "SWA decode metadata missing"
|
||||
elif not all(t is not None and t.is_contiguous() for t in tensors):
|
||||
why = "non-contiguous layer inputs"
|
||||
if why is not None:
|
||||
return self._skip(why)
|
||||
assert swa is not None
|
||||
runner = _mono_runner(x.device)
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
|
||||
)
|
||||
# K1's seam folds the owed post into the residual, except where the
|
||||
# residual arrives settled: the embedding, or after the Engram write
|
||||
fold = True
|
||||
if first:
|
||||
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
|
||||
pre_mix = runner.first_pre_mix(x.shape[0])
|
||||
post_mix = res_mix = pre_mix # unread without the fold
|
||||
fold = False
|
||||
elif layer.engram is not None and engram_hashes is not None:
|
||||
residual = layer.mhc_post(x, residual, post_mix, res_mix)
|
||||
hashes = engram_hashes[:, layer.engram.layer_hash_index]
|
||||
residual = layer.engram(residual, hashes, engram_mask)
|
||||
fold = False
|
||||
kw: dict[str, Any] = {}
|
||||
if attn.indexer is not None:
|
||||
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
|
||||
if attn.compress_ratio:
|
||||
assert comp is not None
|
||||
kw.update(
|
||||
topk_indices=attn.topk_indices_buffer,
|
||||
comp_cache=_record_view(
|
||||
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
|
||||
),
|
||||
comp_block_table=comp.block_table,
|
||||
)
|
||||
return runner.forward(
|
||||
self.weights(layer),
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
positions,
|
||||
swa.slot_mapping,
|
||||
_record_view(attn.swa_cache_layer.kv_cache, swa.block_size),
|
||||
swa.decode_swa_indices,
|
||||
swa.decode_swa_lens,
|
||||
swa.token_to_req_indices,
|
||||
fold=fold,
|
||||
**kw,
|
||||
)
|
||||
|
||||
def ffn(
|
||||
self,
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
part: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_mix: torch.Tensor,
|
||||
res_mix: torch.Tensor,
|
||||
pre_mix: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...] | None:
|
||||
"""The layer's outputs by the FFN launch from ``part``, wo_b's unreduced
|
||||
output, and the attention seam's outputs; None when this step takes the
|
||||
original path (the caller then reduces ``part`` itself)."""
|
||||
why, _, _ = self._decode_step(layer, part.shape[0])
|
||||
tensors = (part, residual, post_mix, res_mix, pre_mix)
|
||||
if why is None and not all(t.is_contiguous() for t in tensors):
|
||||
why = "non-contiguous layer inputs"
|
||||
if why is not None:
|
||||
return self._skip(why)
|
||||
runner = _mono_runner(part.device)
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode FFN launch active (decode steps of <= %d rows)",
|
||||
MAX_ROWS,
|
||||
)
|
||||
return runner.ffn(self.weights(layer), *tensors)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,356 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1 vision variant.
|
||||
|
||||
Thin multimodal wrapper around the text-only ``DeepseekV41LLMForCausalLM``:
|
||||
|
||||
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
|
||||
positions; three learned vectors (``image_start`` / ``image_newline`` /
|
||||
``image_end``) fill the delimiter positions. Every image-span position
|
||||
carries ``image_token_id`` (129264) in ``input_ids``; the per-position
|
||||
roles come from the processor's ``types`` tensor (see
|
||||
``common/mm_preprocess.py``, and ``common/vision.py`` for the tower).
|
||||
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
|
||||
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
|
||||
the model so the MoE router can apply ``bias_vl`` to image tokens
|
||||
(``requires_raw_input_tokens``).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Annotated
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.models.deepseek_v4.common.vision import (
|
||||
DeepseekV4Aligner,
|
||||
DeepseekV4ViT,
|
||||
run_dp_sharded_vision_tower,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from ..common.mm_preprocess import (
|
||||
IMAGE_PLACEHOLDER,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
DeepseekV4VLProcessingInfo,
|
||||
)
|
||||
from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin
|
||||
from ..decoder_replay_layers import DecoderReplayLayers
|
||||
from .model import (
|
||||
DeepseekV41LLMForCausalLM,
|
||||
_linear_scale_param_name,
|
||||
_make_deepseek_v4_weights_mapper,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4VLImagePixelInputs(TensorSchema):
|
||||
"""ViT patch inputs for one batched set of images.
|
||||
|
||||
Dimensions:
|
||||
- np: Total ViT patches across images (sum of n_vit_h * n_vit_w)
|
||||
- c: Number of image channels (3)
|
||||
- p: ViT patch size
|
||||
- ni: Number of images
|
||||
- ns: Total image-span positions (sum of n_llm_h * (n_llm_w + 1) + 2)
|
||||
"""
|
||||
|
||||
patches: Annotated[
|
||||
torch.Tensor, TensorShape("np", 3, "p", "p", dynamic_dims={"np"})
|
||||
]
|
||||
# [n_vit_h, n_vit_w] per image
|
||||
vit_grid: Annotated[torch.Tensor, TensorShape("ni", 2)]
|
||||
# [n_llm_h, n_llm_w] per image
|
||||
llm_grid: Annotated[torch.Tensor, TensorShape("ni", 2)]
|
||||
types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})]
|
||||
|
||||
|
||||
_VL_PREFIX_MAPPING: dict[str, str | None] = {
|
||||
"layers.": "language_model.model.layers.",
|
||||
"embed.": "language_model.model.embed.",
|
||||
"norm.": "language_model.model.norm.",
|
||||
"hc_head": "language_model.model.hc_head",
|
||||
"mtp.": "language_model.model.mtp.",
|
||||
}
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(
|
||||
expert_dtype: str, linear_scale_name: str
|
||||
) -> WeightsMapper:
|
||||
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
|
||||
base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name)
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
|
||||
orig_to_new_regex=base.orig_to_new_regex,
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "language_model.lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
".shared_experts.w2": ".shared_experts.down_proj",
|
||||
# The MTP/DSpark draft heads are not supported for the vision
|
||||
# variant; drop their weights.
|
||||
"mtp.": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
||||
)
|
||||
class DeepseekV41ForCausalLM(
|
||||
nn.Module,
|
||||
DeepseekV4VLEncoderCudaGraphMixin,
|
||||
SupportsMultiModal,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsPP,
|
||||
SupportsEagle3,
|
||||
):
|
||||
"""Multimodal entry point for DeepSeek-V4.1 checkpoints with a vision tower.
|
||||
|
||||
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
|
||||
delegates through ``language_model`` via the protocol defaults.
|
||||
``SupportsEncoderCudaGraph`` (ViT CUDA graph capture/replay) is
|
||||
implemented by ``DeepseekV4VLEncoderCudaGraphMixin``.
|
||||
"""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
# Both of these are read off the *class* by
|
||||
# ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs,
|
||||
# to translate a quantization config's checkpoint-style module names into
|
||||
# runtime prefixes. ``__init__`` later overwrites the instance
|
||||
# ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the
|
||||
# prefix rules matter here. Without the class attributes, per-layer specs
|
||||
# naming ``layers.N.attn.wq_a`` never match this wrapper's
|
||||
# ``language_model.model.layers.N.attn.wq_a`` and silently fall back to
|
||||
# the global spec.
|
||||
packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
|
||||
orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"},
|
||||
)
|
||||
|
||||
# The MoE router needs raw token ids to detect image-span tokens
|
||||
# (all carrying image_token_id, see common/mm_preprocess.py) and apply
|
||||
# bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
return IMAGE_PLACEHOLDER
|
||||
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||
|
||||
def __init__(self, *, vllm_config, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config = model_config.hf_config
|
||||
self.config = config
|
||||
self.multimodal_config = model_config.multimodal_config
|
||||
assert self.multimodal_config is not None
|
||||
|
||||
# The tower is always built; _mark_tower_model stubs it out
|
||||
# (StageMissingLayer, weights skipped) when the image limit is 0.
|
||||
with self._mark_tower_model(vllm_config, {"image"}):
|
||||
self.use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads)
|
||||
self.vision = DeepseekV4ViT(config)
|
||||
self.aligner = DeepseekV4Aligner(config)
|
||||
self.image_start = nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
)
|
||||
self.image_end = nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
)
|
||||
self.image_newline = nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
)
|
||||
self.vision.to(dtype=model_config.dtype)
|
||||
self.aligner.to(dtype=model_config.dtype)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = DeepseekV41LLMForCausalLM(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
)
|
||||
# The outer mapper (see load_weights) fully resolves HF names into
|
||||
# this wrapper's namespace before AutoWeightsLoader strips the
|
||||
# "language_model." prefix and delegates to the child's load_weights,
|
||||
# so the child's own mapper must be a no-op. Its suffix rules are not
|
||||
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
|
||||
# re-fire "head.weight" -> "lm_head.weight").
|
||||
self.language_model.hf_to_vllm_mapper = WeightsMapper()
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
|
||||
expert_dtype, _linear_scale_param_name(vllm_config, expert_dtype)
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> DeepseekV4VLImagePixelInputs | None:
|
||||
patches = kwargs.pop("patches", None)
|
||||
if patches is None:
|
||||
return None
|
||||
return DeepseekV4VLImagePixelInputs(
|
||||
patches=patches,
|
||||
vit_grid=kwargs.pop("vit_grid"),
|
||||
llm_grid=kwargs.pop("llm_grid"),
|
||||
types=kwargs.pop("types"),
|
||||
resolve_bindings={"p": self.config.vision_patch_size},
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
image_input: DeepseekV4VLImagePixelInputs,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
patches = image_input.patches.to(self.aligner.w1.weight.dtype)
|
||||
vit_grid = image_input.vit_grid.tolist()
|
||||
|
||||
image_embeds_list: list[torch.Tensor]
|
||||
if self.use_data_parallel and get_tensor_model_parallel_world_size() > 1:
|
||||
# Data-parallel ViT: shard images across TP ranks and all-gather
|
||||
# the per-image embeddings (weights are replicated on every rank).
|
||||
image_embeds_list = run_dp_sharded_vision_tower(
|
||||
self.vision, self.aligner, patches, vit_grid
|
||||
)
|
||||
else:
|
||||
image_embeds_list = []
|
||||
vit_offset = 0
|
||||
for n_vit_h, n_vit_w in vit_grid:
|
||||
n_vit = n_vit_h * n_vit_w
|
||||
image_embeds_list.append(
|
||||
self._encode_image(
|
||||
patches[vit_offset : vit_offset + n_vit], n_vit_h, n_vit_w
|
||||
)
|
||||
)
|
||||
vit_offset += n_vit
|
||||
|
||||
embeds: list[torch.Tensor] = []
|
||||
span_offset = 0
|
||||
for image_embeds, (n_llm_h, n_llm_w) in zip(
|
||||
image_embeds_list, image_input.llm_grid.tolist(), strict=True
|
||||
):
|
||||
span_len = n_llm_h * (n_llm_w + 1) + 2
|
||||
embeds.append(
|
||||
self._build_image_span(
|
||||
image_embeds,
|
||||
image_input.types[span_offset : span_offset + span_len],
|
||||
)
|
||||
)
|
||||
span_offset += span_len
|
||||
return tuple(embeds)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None:
|
||||
return []
|
||||
return self._process_image_input(image_input)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_model_state_cls():
|
||||
from ..nvidia.model_state import DeepseekV41ModelState
|
||||
|
||||
return DeepseekV41ModelState
|
||||
|
||||
@property
|
||||
def token_lookback_depth(self) -> int:
|
||||
return self.language_model.token_lookback_depth
|
||||
|
||||
@property
|
||||
def decoder_replay_layers(self) -> DecoderReplayLayers | None:
|
||||
return self.language_model.decoder_replay_layers
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
lookback_token_ids: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return self.language_model(
|
||||
input_ids,
|
||||
positions,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
lookback_token_ids=lookback_token_ids,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.language_model.get_expert_mapping()
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map HF names into this wrapper's namespace up front and sort, so
|
||||
# the "language_model." group reaches the child loader as one
|
||||
# contiguous block (AutoWeightsLoader delegates per contiguous group,
|
||||
# and the child's load_weights finalizes fused expert weights, which
|
||||
# must not run on a partially loaded model).
|
||||
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
# The child's load_weights already ran its post-load finalization.
|
||||
self._weights_finalized = True
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Model-level post-load hook (called by the loader after any load
|
||||
# format). Under DummyModelLoader the child's load_weights — and
|
||||
# hence its finalize step — is bypassed, so run it here instead.
|
||||
if getattr(self, "_weights_finalized", False):
|
||||
return
|
||||
self.language_model.process_weights_after_loading()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user