[Model] Add DeepSeek-V4 CPU backend (#55355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-10 14:14:48 +08:00
committed by GitHub
parent 49632fb3ac
commit c3ccc0e957
47 changed files with 9564 additions and 627 deletions
+7
View File
@@ -30,6 +30,13 @@ elif current_platform.is_xpu():
from .xpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
from .xpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
from .xpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
elif current_platform.is_cpu():
from .cpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
from .cpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
from .cpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
from .vl_stub import ( # type: ignore[assignment]
DeepseekV4ForConditionalGeneration,
)
else:
from .nvidia.dspark import ( # type: ignore[assignment]
DSparkDeepseekV4ForCausalLM,
@@ -678,6 +678,19 @@ def fused_indexer_q_rope_quant(
index_q_fp8,
index_weights_out,
)
elif current_platform.is_cpu():
from vllm._custom_ops import fused_indexer_q_rope_quant_cpu
fused_indexer_q_rope_quant_cpu(
positions.contiguous(),
index_q.to(torch.float32),
index_q_cos_sin_cache.to(torch.float32).contiguous(),
index_q_fp8,
index_weights.to(torch.float32),
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out,
)
else:
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
positions,
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 compressor subclass and dispatch: partial-state-cache
write and the compress -> RMSNorm -> RoPE -> FP8 quant -> KV cache store
step, calling ``torch.ops._C.save_partial_states_cpu``/
``compress_norm_rope_store_cpu``/``compress_norm_rope_store_indexer_cpu``
(``csrc/cpu/sgl-kernels/compressor.cpp``) in place of triton-cpu.
Covers both the head_dim=512 main-attention compressor path and the
head_dim=128 indexer compressor path (fp8 only -- MXFP4, ``use_fp4_cache``,
never occurs on CPU: the indexer's MXFP4 cache requires a Blackwell GPU).
"""
from typing import Any, cast
import torch
from vllm._custom_ops import (
compress_norm_rope_store_cpu,
compress_norm_rope_store_indexer_cpu,
save_partial_states_cpu,
)
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.compressor import CompressorMetadata, DeepseekCompressor
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
class DeepseekV4CPUCompressor(DeepseekCompressor):
"""CPU compressor: same state/weights as the shared base, but
``forward``'s cache-write and compress->RMSNorm->RoPE->quant->store
steps always dispatch straight to the ported CPU kernels above instead
of through the shared method's platform-dispatch chain. This class
only ever runs on CPU, and ``use_fp4_cache`` is always ``False`` here
(the indexer's MXFP4 cache requires a Blackwell GPU), so the shared
method's MXFP4/two-stage/cutedsl/triton branches never apply.
"""
_norm_weight_fp32: torch.Tensor
def cache_norm_weight_fp32(self) -> None:
"""Cache a fp32 contiguous copy of ``norm.weight`` once, right after
weight loading (see
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
kernel requires fp32/contiguous but the checkpoint loads this weight
in bf16, and it never changes after loading."""
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
def forward(
self,
kv_score: torch.Tensor,
positions: torch.Tensor,
rotary_emb,
) -> None:
kv, score = kv_score.split(
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
)
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return
state_metadata = attn_metadata[self.state_cache.prefix]
assert isinstance(state_metadata, CompressorMetadata)
token_to_req_indices = state_metadata.token_to_req_indices
assert token_to_req_indices is not None
slot_mapping = state_metadata.slot_mapping
num_actual = slot_mapping.shape[0]
block_table = state_metadata.block_table
block_size = state_metadata.block_size
state_cache = self.state_cache.kv_cache
save_partial_states_cpu(
kv, score, self.ape, positions, state_cache, slot_mapping
)
cos_sin_cache = rotary_emb.cos_sin_cache
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
kv_cache = k_cache_layer.kv_cache
assert self.head_dim in (512, 128), (
f"unsupported compressor head_dim: {self.head_dim}"
)
window = (1 + self.overlap) * self.compress_ratio
win_offsets = torch.arange(window, device=positions.device)
win_local = positions.unsqueeze(-1) - window + 1 + win_offsets
req_idx = token_to_req_indices[:num_actual]
gather_slots = map_local_to_global_slots_cpu(
win_local, req_idx, block_table, block_size
)
kv_cache_2d = kv_cache.view(kv_cache.shape[0], -1)
kv_cache_block_size = kv_cache.shape[1]
store_fn = (
compress_norm_rope_store_cpu
if self.head_dim == 512
else compress_norm_rope_store_indexer_cpu
)
store_fn(
state_cache,
gather_slots,
positions,
k_cache_metadata.slot_mapping,
self._norm_weight_fp32,
self.rms_norm_eps,
cos_sin_cache,
kv_cache_2d,
kv_cache_block_size,
self.compress_ratio,
)
+126
View File
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 sparse-MLA backend descriptor.
DeepSeek-V4 attention runs entirely through
``DeepseekV4Attention``/``DeepseekV4CPUAttention``, never through a generic
``AttentionImpl``. ``DeepseekV4CPUSparseBackend`` exists so the CPU attention
layer has its own backend name and its own metadata-builder class: CPU is not
allowed to run triton-cpu inside the model's forward path, so
``DeepseekV4CPUFlashMLAMetadataBuilder`` overrides the shared base's
``_build_c128a_metadata`` (triton-backed in
``DeepseekV4SparseMLAMetadataBuilder``) with a plain-eager-PyTorch
reimplementation of the same per-token block-table-resolution math.
"""
import torch
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4SparseMLABackend,
DeepseekV4SparseMLAMetadataBuilder,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend,
DeepseekV32IndexerMetadataBuilder,
)
from vllm.v1.attention.backends.mla.sparse_swa import (
_LAYER_TYPE_C4A,
_LAYER_TYPE_C128A,
_LAYER_TYPE_SWAONLY,
DeepseekSparseSWAMetadataBuilder,
)
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta
class DeepseekV4CPUFlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
"""CPU sparse-MLA metadata builder: same fields as the shared base, but
skips the C128A dense-topk metadata entirely -- unlike CUDA/XPU/ROCm,
``DeepseekV4CPUAttention.forward_mqa`` never reads
``c128a_global_decode_topk_indices``/``c128a_decode_topk_lens``/
``c128a_prefill_topk_indices``; it recomputes the same local top-k
directly from ``positions`` instead, so building it here (whether
eagerly or via the shared base's triton kernel) would be wasted work.
"""
def _build_c128a_metadata(
self,
cm: CommonAttentionMetadata,
req_id_per_token: torch.Tensor,
) -> dict[str, torch.Tensor | None]:
return {}
class DeepseekV4CPUSparseBackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "CPU_V4_MLA_SPARSE"
@staticmethod
def get_builder_cls() -> type[DeepseekV4CPUFlashMLAMetadataBuilder]:
return DeepseekV4CPUFlashMLAMetadataBuilder
class DeepseekV4CPUIndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder):
"""CPU indexer metadata builder: same fields as the shared base, but
prefill requests are never split into multiple chunks.
``DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks`` (the
shared base's default) bounds two things the CUDA/XPU indexer kernels
need: the flat-gather workspace size and the dense M*N logits tensor
those kernels allocate for a chunk. The CPU indexer
(``sparse_attn_indexer_cpu``) allocates neither -- it reads the paged
K-cache directly via ``fp8_paged_mqa_logits_cpu``/
``topk_transform_512_cpu``, with no chunk-splitting of its own.
With chunked prefill now enabled for this model on CPU (see
``CpuPlatform.check_and_update_config``'s ``amx_mla_or_dsv4_enabled``),
the total query-token count for one step is already bounded by
``max_num_batched_tokens`` -- there is nothing left here to bound, so
this always returns a single chunk spanning the whole step's prefill
batch.
"""
@staticmethod
def _split_indexer_prefill_chunks(
compressed_seq_lens_cpu: torch.Tensor,
prefill_query_lens_cpu: torch.Tensor,
workspace_size: int,
max_logits_bytes: int,
request_offset: int = 0,
) -> list[tuple[slice, slice]]:
# workspace_size/max_logits_bytes are unused: no flat-gather
# workspace or dense M*N buffer is allocated.
num_reqs = compressed_seq_lens_cpu.shape[0]
total_query_len = int(prefill_query_lens_cpu.sum().item())
return [
(
slice(request_offset, request_offset + num_reqs),
slice(0, total_query_len),
)
]
class DeepseekV4CPUIndexerBackend(DeepseekV4IndexerBackend):
@staticmethod
def get_name() -> str:
return "CPU_V4_INDEXER"
@staticmethod
def get_builder_cls() -> type[DeepseekV4CPUIndexerMetadataBuilder]:
return DeepseekV4CPUIndexerMetadataBuilder
class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
"""CPU SWA metadata builder: same fields as the shared base."""
def build_tile_scheduler(
self, num_decode_tokens: int
) -> dict[str, FlashMLASchedMeta | None]:
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
C++ decode-path concern) -- always return the all-``None`` sentinel
the shared base's own CPU branch returns."""
return {
_LAYER_TYPE_SWAONLY: None,
_LAYER_TYPE_C4A: None,
_LAYER_TYPE_C128A: None,
}
+648
View File
@@ -0,0 +1,648 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 attention subclass.
``forward_mqa`` resolves SWA/compressed top-k indices to paged-cache slot
ids and calls the fused ``flash_mla_with_kvcache_cpu`` kernel. ``_o_proj``
stays eager -- not on the attention hot path.
"""
from typing import Any, cast
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import (
DeepseekV4Attention,
DeepseekV4Indexer,
DeepseekV4IndexerCache,
)
from vllm.models.deepseek_v4.cpu.cpu_compressor import DeepseekV4CPUCompressor
from vllm.models.deepseek_v4.cpu.cpu_mla import (
DeepseekV4CPUIndexerBackend,
DeepseekV4CPUSparseBackend,
)
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_ROPE_DIM = 64
def _deepseek_v4_cpu_prepare_and_attn(
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor | None,
indexer_kv_score: torch.Tensor | None,
indexer_weights: torch.Tensor | None,
positions: torch.Tensor,
o_padded: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
"""Opaque custom-op boundary around attention-prep + sparse-indexer +
MLA attention + output projection -- CPU-only.
Wraps cache writes with no explicit tensor args (SWA/compressor/indexer
state), which torch.compile can't otherwise track as mutations -- one
opaque call sidesteps that. ``_o_proj`` is folded in too because CPU's
``DYNAMO_TRACE_ONCE`` mode traces exactly once, during warmup when
``attn_metadata`` is ``None``; tracing ``_o_proj`` separately would bake
that warmup branch into the compiled graph permanently, dead-code-
eliminating the whole attention computation upstream.
"""
layer_name_str = _resolve_layer_name(layer_name)
self = get_forward_context().no_compile_layers[layer_name_str]
self._prepare_and_attn_fn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
)
o = o_padded[:, : self.n_local_heads, :]
return self._o_proj(o, positions)
def _deepseek_v4_cpu_prepare_and_attn_fake(
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor | None,
indexer_kv_score: torch.Tensor | None,
indexer_weights: torch.Tensor | None,
positions: torch.Tensor,
o_padded: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
# hidden_states' last dim already equals _o_proj's output width, so we
# don't need to resolve layer_name (an opaque object) here.
return hidden_states.new_empty((hidden_states.shape[0], hidden_states.shape[-1]))
direct_register_custom_op(
op_name="deepseek_v4_cpu_prepare_and_attn",
op_func=_deepseek_v4_cpu_prepare_and_attn,
mutates_args=["o_padded"],
fake_impl=_deepseek_v4_cpu_prepare_and_attn_fake,
)
def _dequant_linear_weight(layer: torch.nn.Module) -> torch.Tensor:
"""Dequantize a linear layer's weight, including FP8 block scales.
``_o_proj`` reads ``wo_a.weight`` directly (not via
``quant_method.apply()``), so the per-block FP8 scale must be applied
here explicitly.
"""
weight = layer.weight
scale = getattr(layer, "weight_scale_inv", None)
if scale is None:
scale = getattr(layer, "weight_scale", None)
if scale is None:
return weight.float()
if scale.dtype == torch.float8_e8m0fnu:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
scale = _upcast_e8m0_to_fp32(scale)
scale = scale.to(torch.float32)
if scale.numel() == 1:
return weight.float() * scale
block_size = getattr(layer, "weight_block_size", None)
if block_size is None:
# Per-channel scale: one row-scale per output row.
return weight.float() * scale.view(-1, 1)
block_m, block_k = block_size
m, k = weight.shape
scale = scale.repeat_interleave(block_m, dim=0).repeat_interleave(block_k, dim=1)
return weight.float() * scale[:m, :k]
def _fused_indexer_q_rope_quant_cpu(
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""CPU-only equivalent of ``fused_indexer_q_rope_quant``'s FP8 path
(CPU never reaches its MXFP4 arm -- Blackwell-only). Returns
``(q_fp8, weights_out)``, folding the per-token q scale into
``index_weights``. ``positions``/``index_q_cos_sin_cache`` are already
contiguous int64/fp32, so no conversion is needed here.
"""
assert positions.ndim == 1
assert index_q.ndim == 3
assert index_q_cos_sin_cache.ndim == 2
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
index_q_fp8 = torch.empty_like(index_q, dtype=current_platform.fp8_dtype())
ops.fused_indexer_q_rope_quant_cpu(
positions,
index_q.to(torch.float32),
index_q_cos_sin_cache,
index_q_fp8,
index_weights.to(torch.float32),
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out,
)
return index_q_fp8, index_weights_out
class _NoOpEvent:
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
stream list, these events are constructed but never actually used."""
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
"""CPU indexer K-cache descriptor: same fields as the shared base, just
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV4CPUIndexerBackend
class DeepseekV4CPUIndexer(DeepseekV4Indexer):
"""CPU indexer: the C4A short-context fallback runs as eager PyTorch
instead of Triton. Never constructed directly -- ``__class__`` is
swapped onto an existing ``DeepseekV4Indexer`` instance post-construction.
"""
def forward(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
compressed_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
qr_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
"""CPU override: no aux streams, so wq_b_and_q_quant and the
compressor run straight-line instead of through
``maybe_execute_in_parallel``; the short-context check drops the
CUDA-only ``is_current_stream_capturing()`` guard (always False
here)."""
compressor = self.compressor
attn_metadata = get_forward_context().attn_metadata
if isinstance(attn_metadata, dict):
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
if indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens:
# Fewer candidates than topk: all are selected, but the K
# cache still needs to be built.
compressor(compressed_kv_score, positions, rotary_emb)
assert self.topk_indices_buffer is not None
num_tokens = (
indexer_metadata.num_decode_tokens
+ indexer_metadata.num_prefill_tokens
)
if num_tokens > 0:
self._fill_short_context_topk(positions, num_tokens)
return None, None, None
def wq_b_and_q_quant():
q = self._wq_b_proj(qr, qr_scale)
q = q.view(-1, self.n_head, self.head_dim)
return _fused_indexer_q_rope_quant_cpu(
positions,
q,
rotary_emb.cos_sin_cache,
indexer_weights,
self.softmax_scale,
self.n_head**-0.5,
)
# Order matches the base class's parallel-join contract: compressor
# must complete before indexer_op runs (skip_k_cache_insert=True).
q, weights = wq_b_and_q_quant()
compressor(compressed_kv_score, positions, rotary_emb)
# No q_scale: FP8-only path (MXFP4 needs Blackwell).
return q, None, weights
def _fill_short_context_topk(
self, positions: torch.Tensor, num_tokens: int
) -> None:
assert self.topk_indices_buffer is not None
device = positions.device
arange = torch.arange(self.topk_tokens, device=device)
num_compressed = torch.div(
positions[:num_tokens] + 1, self.compress_ratio, rounding_mode="floor"
)
valid = arange.unsqueeze(0) < num_compressed.unsqueeze(-1)
filled = torch.where(
valid, arange.unsqueeze(0), torch.full_like(arange.unsqueeze(0), -1)
)
self.topk_indices_buffer[:num_tokens, : self.topk_tokens] = filled.to(
self.topk_indices_buffer.dtype
)
class DeepseekV4CPUAttention(DeepseekV4Attention):
"""CPU sparse MLA attention layer for DeepSeek V4."""
backend_cls = DeepseekV4CPUSparseBackend
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
# Swap classes post-construction (no class-swap hook on the shared
# base) so forward()/get_attn_backend() dispatch to CPU kernels.
# SWA-only layers (compress_ratio <= 1) never build a
# compressor/indexer.
if self.compressor is not None:
self.compressor.__class__ = DeepseekV4CPUCompressor
if self.indexer is not None:
self.indexer.__class__ = DeepseekV4CPUIndexer
self.indexer.k_cache.__class__ = DeepseekV4CPUIndexerCache
self.indexer.compressor.__class__ = DeepseekV4CPUCompressor
self._wo_a_packed: torch.Tensor | None = None
self._wrap_wo_a_process_weights_after_loading()
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
"""Cache fp32-contiguous copies of tensors CPU kernels want that way
but that arrive bf16 (rotary cos/sin table, compressor RMSNorm
weights) -- cast once here instead of per forward call.
Runs after every quantized layer's own
``process_weights_after_loading`` (see ``is_deferred_attention_layer``);
``wo_a``'s packing must happen *before* that phase instead, so it's a
separate monkeypatch in ``__init__``.
"""
self.rotary_emb.cos_sin_cache = self.rotary_emb.cos_sin_cache.to(
torch.float32
).contiguous()
if self.compressor is not None:
self.compressor.cache_norm_weight_fp32()
if self.indexer is not None:
self.indexer.compressor.cache_norm_weight_fp32()
def _wrap_wo_a_process_weights_after_loading(self) -> None:
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
``bmm_cpu`` before the FP8 kernel's own
``process_weights_after_loading`` VNNI-repacks it in place for
row-major reads ``_o_proj`` never does."""
wo_a = self.wo_a
quant_method = wo_a.quant_method
orig_pwal = quant_method.process_weights_after_loading
heads_per_group = self.n_local_heads // self.n_local_groups
k_dim = heads_per_group * self.head_dim
def _capture_and_pack(layer: torch.nn.Module) -> None:
dequant = _dequant_linear_weight(layer).to(torch.bfloat16)
dequant = dequant.view(self.n_local_groups, self.o_lora_rank, k_dim)
self._wo_a_packed = torch.ops._C.convert_weight_packed(dequant.contiguous())
orig_pwal(layer)
# orig_pwal VNNI-repacks layer.weight/scale for ordinary-linear
# use that _o_proj never needs -- free them, same pattern as
# dispatch_cpu_unquantized_gemm's remove_weight=True.
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
scale_attr = (
"weight_scale_inv"
if hasattr(layer, "weight_scale_inv")
else "weight_scale"
)
if hasattr(layer, scale_attr):
setattr(
layer,
scale_attr,
torch.nn.Parameter(torch.empty(0), requires_grad=False),
)
quant_method.process_weights_after_loading = ( # type: ignore[method-assign]
_capture_and_pack
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
"""CPU override: wraps attention-prep + MLA + output projection in the
CPU-only opaque custom op (see ``_deepseek_v4_cpu_prepare_and_attn``).
"""
num_tokens = hidden_states.shape[0]
o_padded = torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
kv_score: torch.Tensor | None = None
if self.compressor is not None:
# Must go through the module's own forward (not a raw matmul on
# its weight): the packed-GEMM kernel frees the raw weight after
# packing, same as wo_a's.
kv_score = self.compressor.fused_wkv_wgate(hidden_states).to(torch.float32)
indexer_weights: torch.Tensor | None = None
indexer_kv_score: torch.Tensor | None = None
if self.indexer is not None:
indexer_weights, _ = self.indexer.weights_proj(hidden_states)
indexer_kv_score = self.indexer.compressor.fused_wkv_wgate(
hidden_states
).to(torch.float32)
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
return torch.ops.vllm.deepseek_v4_cpu_prepare_and_attn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
_encode_layer_name(self.prefix),
)
def _fused_qnorm_rope_kv_insert(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
attn_metadata: (
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
),
) -> torch.Tensor:
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
supported here, so the base method's bf16/per-tensor-fp8 branches
are dropped."""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
if self.n_local_heads < self.padded_heads:
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
return q
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_kv_cache = self.swa_cache_layer.kv_cache
# positions is already int64 (the runner's buffer), no cast needed.
assert positions.dtype == torch.int64
assert swa_kv_cache.dtype == torch.uint8, (
"DeepseekV4CPUAttention only supports the fp8_ds_mla cache layout"
)
# Fused: Q side does per-head weight-free RMSNorm + GPT-J RoPE
# (zero-filling padded head slots); KV side does GPT-J RoPE + UE8M0
# FP8 quant + paged insert.
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
return ops.fused_qnorm_rope_kv_insert_cpu(
q,
kv,
positions,
swa_kv_cache_2d,
swa_metadata.slot_mapping,
self.rotary_emb.cos_sin_cache,
self.padded_heads,
self.eps,
swa_metadata.block_size,
)
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""CPU override: two ``RMSNorm`` calls instead of the shared
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
return self.q_norm(qr), None, self.kv_norm(kv)
def _prepare_and_attn(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
) -> None:
"""CPU override: no aux streams, so query-projection+KV-insert, the
indexer, and the compressor run straight-line instead of through
execute_in_parallel/maybe_execute_in_parallel."""
attn_metadata = get_forward_context().attn_metadata
indexer = self.indexer
compressor = self.compressor
q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_local_heads, self.head_dim)
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
index_q: torch.Tensor | None = None
index_q_scale: torch.Tensor | None = None
index_weights_out: torch.Tensor | None = None
if indexer is not None:
assert compressor is not None
index_q, index_q_scale, index_weights_out = indexer(
hidden_states,
qr,
indexer_kv_score,
indexer_weights,
positions,
self.indexer_rotary_emb,
qr_scale,
)
compressor(kv_score, positions, self.rotary_emb)
elif compressor is not None:
compressor(kv_score, positions, self.rotary_emb)
self._sparse_indexer_and_attn(
hidden_states,
index_q,
index_q_scale,
index_weights_out,
q,
kv,
positions,
o_padded,
)
def _sparse_indexer_and_attn(
self,
hidden_states: torch.Tensor,
index_q: torch.Tensor | None,
index_q_scale: torch.Tensor | None,
index_weights: torch.Tensor | None,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor,
) -> None:
"""CPU override: identical body, minus ``@eager_break_during_capture``
(this platform never captures a CUDA graph, so it was a no-op)."""
if self.indexer is not None and index_q is not None:
assert index_weights is not None
# index_q_scale is always None on CPU (FP8-only path).
assert index_q_scale is None
self.indexer.indexer_op(hidden_states, index_q, None, index_weights)
self.forward_mqa(q, kv, positions, out)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
# No head-count padding constraint for the (not yet ported) CPU kernel.
return num_heads
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
if get_forward_context().attn_metadata is None:
# Warmup/profile run: no real metadata, o is a zeroed placeholder.
return torch.zeros(
(o.shape[0], self.hidden_size), dtype=o.dtype, device=o.device
)
num_tokens, num_heads, _ = o.shape
cos_sin_cache = self.rotary_emb.cos_sin_cache
# Undo the query's own GPT-J RoPE rotation: the attention output is
# a weighted sum of KV rows at different positions, so R(-pos_q)
# recovers the translation-invariant quantity wo_a/wo_b expect.
o_derot = ops.inverse_gptj_rope_o_proj_cpu(
o, positions, cos_sin_cache, _ROPE_DIM
)
heads_per_group = num_heads // self.n_local_groups
o_grouped = o_derot.reshape(
num_tokens, self.n_local_groups, heads_per_group * self.head_dim
).to(torch.bfloat16)
assert self._wo_a_packed is not None
z = o_grouped.new_empty((self.n_local_groups, num_tokens, self.o_lora_rank))
ops.bmm_cpu(
z,
o_grouped.transpose(0, 1).contiguous(),
self._wo_a_packed,
True,
None,
)
z = z.transpose(0, 1).to(o.dtype)
return self.wo_b(z.flatten(1))
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# Warmup/profile run: reserve nothing extra, skip real kernels.
output.zero_()
return
assert isinstance(attn_metadata, dict)
swa_metadata = cast(
DeepseekSparseSWAMetadata,
attn_metadata.get(self.swa_cache_layer.prefix),
)
assert swa_metadata is not None
assert swa_metadata.token_to_req_indices is not None
req_idx = swa_metadata.token_to_req_indices
swa_only = self.compress_ratio <= 1
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
)
num_tokens = q.shape[0]
device = q.device
# ---- SWA window: causal sliding window over the raw per-token cache
window = self.window_size
win_offsets = torch.arange(window, device=device)
start_pos = (positions - window + 1).clamp(min=0)
local_pos = start_pos.unsqueeze(-1) + win_offsets
valid_win = local_pos <= positions.unsqueeze(-1)
win_local = torch.where(valid_win, local_pos, torch.full_like(local_pos, -1))
window_slots = map_local_to_global_slots_cpu(
win_local, req_idx, swa_metadata.block_table, swa_metadata.block_size
)
window_cache_2d = self.swa_cache_layer.kv_cache.view(
self.swa_cache_layer.kv_cache.shape[0], -1
)
# ---- Compressed KV: C4A's learned top-k, or C128A's dense top-k ----
if swa_only:
compressed_slots = window_slots.new_full((num_tokens, 0), -1)
# Unused when num_compressed == 0 (only dtype/2D-ness matter);
# reuse the window cache to avoid an allocation.
compressed_cache_2d = window_cache_2d
compressed_block_size = 1
else:
assert flashmla_metadata is not None
main_block_size = flashmla_metadata.block_size // self.compress_ratio
if self.compress_ratio == 4:
assert self.topk_indices_buffer is not None
local_topk = self.topk_indices_buffer[:num_tokens]
else:
width = -(-self.max_model_len // self.compress_ratio)
num_compressed = torch.div(
positions + 1, self.compress_ratio, rounding_mode="floor"
).clamp(max=width)
arange_w = torch.arange(width, device=device)
local_topk = torch.where(
arange_w.unsqueeze(0) < num_compressed.unsqueeze(-1),
arange_w.unsqueeze(0).expand(num_tokens, width),
torch.full(
(num_tokens, width), -1, device=device, dtype=torch.int64
),
)
compressed_slots = map_local_to_global_slots_cpu(
local_topk, req_idx, flashmla_metadata.block_table, main_block_size
)
compressed_cache_2d = self.kv_cache.view(self.kv_cache.shape[0], -1)
compressed_block_size = main_block_size
torch.ops._C.flash_mla_with_kvcache_cpu(
output,
q,
window_cache_2d,
window_slots,
swa_metadata.block_size,
compressed_cache_2d,
compressed_slots,
compressed_block_size,
self.attn_sink,
self.scale,
)
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for DeepSeek-V4's CPU-ported kernels."""
import torch
def map_local_to_global_slots_cpu(
local_indices: torch.Tensor,
req_idx: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
) -> torch.Tensor:
"""Map per-request-local (token or compressed-token) indices to global
paged-cache slot ids via a block table, mirroring
``compute_global_topk_indices_and_lens``.
Args:
local_indices: [N, K], -1 sentinel for invalid entries.
req_idx: [N], row into ``block_table`` for each of the N rows.
block_table: [num_reqs, max_blocks_per_seq].
block_size: tokens (or compressed tokens) per physical block.
Returns:
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
"""
valid = local_indices >= 0
safe_local = local_indices.clamp(min=0).to(torch.int64)
block_pos = torch.div(safe_local, block_size, rounding_mode="floor")
offset = safe_local % block_size
req_expand = req_idx.to(torch.int64).unsqueeze(-1).expand_as(local_indices)
block_num = block_table[req_expand, block_pos].to(torch.int64)
slot = block_num * block_size + offset
return torch.where(valid, slot, torch.full_like(slot, -1))
+18
View File
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 DSpark (speculative decoding) — deferred, not implemented.
``DSparkDeepseekV4ForCausalLM`` is only instantiated when the speculative-
decoding registry resolves it by name for a DSpark speculative config; a
plain (non-spec) DeepSeek-V4 CPU model never constructs this class. This
stub exists solely so ``vllm.models.deepseek_v4.cpu`` is import-clean.
"""
from vllm.config import VllmConfig
class DSparkDeepseekV4ForCausalLM:
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
raise NotImplementedError(
"DeepSeek-V4 speculative decoding (DSpark) is not implemented on CPU."
)
+922
View File
@@ -0,0 +1,922 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
from itertools import islice
import regex as re
import torch
import torch.nn as nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
HAS_TILELANG_MHC,
HCHeadOp,
MHCFusedPostPreOp,
MHCPostOp,
MHCPreOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
SupportsEagle3,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
from vllm.models.deepseek_v4.cpu.cpu_sparse import DeepseekV4CPUAttention
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
class DeepseekV4MLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
swiglu_limit: float | None = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
is_sequence_parallel: bool = False,
prefix: str = "",
) -> None:
super().__init__()
# is_sequence_parallel: weights are replicated, no collective ops
# needed. Otherwise: standard TP with an allreduce at the end.
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
disable_tp=is_sequence_parallel,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
disable_tp=is_sequence_parallel,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
if swiglu_limit is not None:
self.act_fn = SiluAndMulWithClamp(swiglu_limit)
else:
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class DeepseekV4MoE(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.prefix = prefix
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
self.hidden_size = config.hidden_size
self.n_routed_experts = config.n_routed_experts
self.n_activated_experts = config.num_experts_per_tok
self.moe_intermediate_size = config.moe_intermediate_size
self.swiglu_limit = config.swiglu_limit
self.renormalize = config.norm_topk_prob
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
self.gate = GateLinear(
input_size=config.hidden_size,
output_size=config.n_routed_experts,
bias=False,
out_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
# Use randint instead of empty to avoid garbage values causing
# invalid memory access in dummy mode (--load-format="dummy")
self.gate.tid2eid = nn.Parameter(
torch.randint(
0,
config.n_routed_experts,
(config.vocab_size, config.num_experts_per_tok),
dtype=self.hash_indices_dtype,
),
requires_grad=False,
)
elif getattr(config, "topk_method", None) == "noaux_tc":
self.gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.n_routed_experts, dtype=torch.float32),
requires_grad=False,
)
self.n_shared_experts = config.n_shared_experts
# No CPU fused-shared-expert grouped-GEMM path yet.
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
self.shared_experts = DeepseekV4MLP(
hidden_size=config.hidden_size,
intermediate_size=intermediate_size,
hidden_act=config.hidden_act,
swiglu_limit=self.swiglu_limit,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.shared_experts",
)
self.tp_rank = get_tensor_model_parallel_rank()
assert config.n_routed_experts % self.tp_size == 0
self.n_local_experts = config.n_routed_experts // self.tp_size
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
n_shared_experts=(
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
prefix=f"{prefix}.experts",
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
e_score_correction_bias=self.gate.e_score_correction_bias,
hash_indices_table=self.gate.tid2eid,
swiglu_limit=self.swiglu_limit,
router_logits_dtype=torch.float32,
)
if is_hash_moe:
# apply_monolithic() never sees hash_indices_table via the
# router path (modular-only) -- stash it on RoutedExperts
# directly. object.__setattr__ avoids nn.Module re-registering
# (and duplicating in the state_dict) this alias.
object.__setattr__(
self.experts.routed_experts, "hash_indices_table", self.gate.tid2eid
)
def forward(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
) -> torch.Tensor:
if self.gate.tid2eid is not None and input_ids is None:
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
org_shape = hidden_states.shape
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
input_ids=input_ids,
)
return final_hidden_states.view(org_shape)
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
vllm_config,
prefix,
topk_indices_buffer: torch.Tensor | None = None,
):
super().__init__()
# Lazy import to avoid top-level tilelang dependency.
# Registers both torch.ops.vllm.mhc_pre and mhc_post
import vllm.model_executor.layers.mhc # noqa: F401
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.rms_norm_eps = config.rms_norm_eps
self.attn = DeepseekV4CPUAttention(
vllm_config,
prefix=f"{prefix}.attn",
topk_indices_buffer=topk_indices_buffer,
)
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.hc_mult = config.hc_mult
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
self.hc_eps = config.hc_eps
self.hc_post_alpha = 2.0
mix_hc = (2 + self.hc_mult) * self.hc_mult
hc_dim = self.hc_mult * self.hidden_size
self.hc_attn_fn = nn.Parameter(
torch.empty(
(mix_hc, hc_dim),
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_fn = nn.Parameter(
torch.empty(
(mix_hc, hc_dim),
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_attn_base = nn.Parameter(
torch.empty(
mix_hc,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_base = nn.Parameter(
torch.empty(
mix_hc,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_attn_scale = nn.Parameter(
torch.empty(
3,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_ffn_scale = nn.Parameter(
torch.empty(
3,
dtype=torch.float32,
),
requires_grad=False,
)
self.mhc_pre = MHCPreOp()
self.mhc_post = MHCPostOp()
self.mhc_fused_post_pre = MHCFusedPostPreOp()
self.use_fused_mhc = HAS_TILELANG_MHC and not (
HAS_AITER_MHC and self.hidden_size % 256 == 0
)
def hc_pre(
self,
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
):
post_mix, res_mix, layer_input = self.mhc_pre(
residual=x,
fn=hc_fn,
hc_scale=hc_scale,
hc_base=hc_base,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
)
return layer_input, post_mix, res_mix
def hc_post(
self,
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
):
return self.mhc_post(x, residual, post, comb)
def _forward_fused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if residual is None:
# Run standalone hc_pre on first layer
residual = x
x, post_mix, res_mix = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
else:
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
def _forward_unfused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
]:
residual = x
x, post, comb = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x, None, None, None
def forward(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
]:
if not self.use_fused_mhc:
return self._forward_unfused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
return self._forward_fused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.vocab_size = config.vocab_size
self.hc_eps = config.hc_eps
self.hc_mult = config.hc_mult
self.hc_dim = self.hc_mult * config.hidden_size
self.rms_norm_eps = config.rms_norm_eps
self.device = current_platform.device_type
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=self.device,
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV4DecoderLayer(
vllm_config,
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV4MoE,
"ffn",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.hc_head_fn = nn.Parameter(
torch.empty(
self.hc_mult,
self.hc_dim,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_head_base = nn.Parameter(
torch.empty(
self.hc_mult,
dtype=torch.float32,
),
requires_grad=False,
)
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_op = HCHeadOp()
spec_config = vllm_config.speculative_config
needs_mtp_hidden_states = spec_config is not None and (
spec_config.use_eagle() or spec_config.uses_draft_model()
)
if get_pp_group().is_last_rank and needs_mtp_hidden_states:
self._mtp_hidden_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
self.hc_dim,
dtype=vllm_config.model_config.dtype,
device=self.device,
)
else:
self._mtp_hidden_buffer = None
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def make_empty_intermediate_tensors(
self,
batch_size: int,
dtype: torch.dtype,
device: torch.device,
) -> IntermediateTensors:
# PP intermediate tensors keep the (num_tokens, hc_mult, hidden_size)
# multi-stream shape until hc_head() collapses it.
return IntermediateTensors(
{
"hidden_states": torch.zeros(
(batch_size, self.hc_mult, self.config.hidden_size),
dtype=dtype,
device=device,
),
}
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
hidden_states = hidden_states.unsqueeze(-2).repeat(1, self.hc_mult, 1)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual, post_mix, res_mix = None, None, None
# EAGLE3/DSpark/DFlash aux hidden states: post-mhc hidden state at
# each target layer, averaged over hc_mult streams to [T,
# hidden_size]. Empty unless a draft model set aux_hidden_state_layers.
aux_hidden_states: list[torch.Tensor] = []
# Reused below to avoid computing hc_post twice if the last layer is
# also an aux layer.
final_aux_recon: torch.Tensor | None = None
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
)
if (idx + 1) in self.aux_hidden_state_layers:
# Unfused path already applied hc_post; fused path needs it
# applied here before averaging.
if layer.use_fused_mhc:
aux_recon = layer.hc_post(
hidden_states, residual, post_mix, res_mix
)
final_aux_recon = aux_recon
else:
aux_recon = hidden_states
aux_hidden_states.append(aux_recon.mean(dim=1))
if layer is not None and layer.use_fused_mhc:
# Reuse the last layer's hc_post output if already computed above.
if (
final_aux_recon is not None
and self.end_layer in self.aux_hidden_state_layers
):
hidden_states = final_aux_recon
else:
hidden_states = layer.hc_post(
hidden_states, residual, post_mix, res_mix
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
if self._mtp_hidden_buffer is not None:
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
hidden_states = self.hc_head_op(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
self.hc_head_base,
self.rms_norm_eps,
self.hc_eps,
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "w1", 0),
("gate_up_proj", "w3", 1),
("attn.fused_wqa_wkv", "attn.wq_a", 0),
("attn.fused_wqa_wkv", "attn.wkv", 1),
("compressor.fused_wkv_wgate", "compressor.wkv", 0),
("compressor.fused_wkv_wgate", "compressor.wgate", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _resolve_param_name(name: str) -> str:
inv_name = f"{name}_inv"
if name not in params_dict and inv_name in params_dict:
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
# Each MoE's own per-layer fusion decision is the source of truth,
# so the redirect below can't diverge from how it was built.
fuse_by_layer = {
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
for mod_name, mod in self.named_modules()
if isinstance(mod, DeepseekV4MoE)
}
n_routed = self.config.n_routed_experts
# Maps the single shared-expert tensor group to one appended slot;
# multiple shared experts would need per-expert slicing (see
# deepseek_v2.py). DeepSeek-V4 has n_shared_experts == 1.
if any(fuse_by_layer.values()) and self.config.n_shared_experts != 1:
raise NotImplementedError(
"deepseek-v4 fused shared-expert loading supports only "
f"n_shared_experts == 1, got {self.config.n_shared_experts}"
)
for name, loaded_weight in weights:
# Shared-expert fusion: redirect shared_experts.w{1,2,3} into
# the appended routed-expert slot so it loads through the
# routed expert (grouped GEMM) loader.
if ".ffn.shared_experts.w" in name and fuse_by_layer.get(
extract_layer_index(name), False
):
name = name.replace(
".ffn.shared_experts.w",
f".ffn.experts.{n_routed}.w",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
continue
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
break
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(param_name)
break
else:
if ".experts." in name:
# E8M0 scales are float8_e8m0fnu in checkpoints but the
# MoE param is uint8; copy_() would numerically convert
# (e.g. 2^-7 -> 0) instead of preserving raw bytes.
if (
"weight_scale" in name
and loaded_weight.dtype == torch.float8_e8m0fnu
):
loaded_weight = loaded_weight.view(torch.uint8)
for mapping in expert_mapping:
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
# return_success lets us skip experts that have
# another replica available.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(param_name)
continue
return loaded_params
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# (param_name, weight_name, expert_id, shard_id). When fusing shared
# experts, include the appended slots so redirected shared-expert
# weights route through the expert loader.
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.n_routed_experts + (
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=num_experts,
)
def _make_deepseek_v4_weights_mapper(
expert_dtype: str, fuse_shared_experts: bool = False
) -> WeightsMapper:
if expert_dtype == "fp4":
# MXFP4 experts register scales as ``w{1,2,3}_weight_scale`` (no
# _inv); FP8 linear/shared-expert scales register as
# ``weight_scale_inv``. Route DeepSeek-native ``.scale`` and
# AMD-Quark ``.weight_scale`` keys accordingly, leaving expert/
# shared-expert ``w{1,2,3}.weight_scale`` untouched for the MXFP4
# loader.
scale_regex = {
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
re.compile(r"\.scale$"): ".weight_scale_inv",
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
}
else:
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
# scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys
# there.
scale_regex = {
re.compile(r"\.scale$"): ".weight_scale_inv",
}
# Fused shared experts are redirected to routed expert slots elsewhere;
# leave their names untouched here.
substr_map: dict[str, str | None] = (
{}
if fuse_shared_experts
else {".shared_experts.w2": ".shared_experts.down_proj"}
)
# No MTP support on CPU yet: drop mtp.* weights (no model.mtp submodule).
substr_map["mtp."] = None
return WeightsMapper(
orig_to_new_prefix={
"layers.": "model.layers.",
"embed.": "model.embed.",
"norm.": "model.norm.",
"hc_head": "model.hc_head",
"mtp.": "model.mtp.",
},
orig_to_new_regex=scale_regex,
orig_to_new_suffix={
"head.weight": "lm_head.weight",
"embed.weight": "embed_tokens.weight",
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
".input_scale": ".input_scale_2",
},
orig_to_new_substr=substr_map,
)
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
model_cls = DeepseekV4Model
# Default mapper assumes the original FP4-expert checkpoint layout.
# Overridden per-instance in __init__ when expert_dtype != "fp4".
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
packed_modules_mapping = {
"gate_up_proj": ["w1", "w3"],
"fused_wqa_wkv": ["wq_a", "wkv"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
expert_dtype = getattr(config, "expert_dtype", "fp4")
self.model = self.model_cls(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype,
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
hc_mult * hidden_size) for the MTP draft model. Populated by
forward(); valid after each target step."""
return getattr(self.model, "_mtp_hidden_buffer", None)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
# No CPU AMX weight-packing hook yet.
pass
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+19
View File
@@ -0,0 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU DeepSeek-V4 MTP (speculative decoding) — deferred, not implemented.
``DeepSeekV4MTP`` is only instantiated when
``vllm/model_executor/models/registry.py``'s speculative-decoding registry
resolves it by name for a speculative-decoding config; a plain (non-spec)
DeepSeek-V4 CPU model never constructs this class. This stub exists solely
so ``vllm.models.deepseek_v4.cpu`` is import-clean.
"""
from vllm.config import VllmConfig
class DeepSeekV4MTP:
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
raise NotImplementedError(
"DeepSeek-V4 speculative decoding (MTP) is not implemented on CPU."
)