[Model] Add DeepSeek-V4 CPU backend (#55355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -30,6 +30,13 @@ elif current_platform.is_xpu():
|
||||
from .xpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .xpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .xpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
elif current_platform.is_cpu():
|
||||
from .cpu.dspark import DSparkDeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .cpu.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .cpu.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
from .vl_stub import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
else:
|
||||
from .nvidia.dspark import ( # type: ignore[assignment]
|
||||
DSparkDeepseekV4ForCausalLM,
|
||||
|
||||
@@ -678,6 +678,19 @@ def fused_indexer_q_rope_quant(
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
from vllm._custom_ops import fused_indexer_q_rope_quant_cpu
|
||||
|
||||
fused_indexer_q_rope_quant_cpu(
|
||||
positions.contiguous(),
|
||||
index_q.to(torch.float32),
|
||||
index_q_cos_sin_cache.to(torch.float32).contiguous(),
|
||||
index_q_fp8,
|
||||
index_weights.to(torch.float32),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
else:
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
|
||||
positions,
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 compressor subclass and dispatch: partial-state-cache
|
||||
write and the compress -> RMSNorm -> RoPE -> FP8 quant -> KV cache store
|
||||
step, calling ``torch.ops._C.save_partial_states_cpu``/
|
||||
``compress_norm_rope_store_cpu``/``compress_norm_rope_store_indexer_cpu``
|
||||
(``csrc/cpu/sgl-kernels/compressor.cpp``) in place of triton-cpu.
|
||||
|
||||
Covers both the head_dim=512 main-attention compressor path and the
|
||||
head_dim=128 indexer compressor path (fp8 only -- MXFP4, ``use_fp4_cache``,
|
||||
never occurs on CPU: the indexer's MXFP4 cache requires a Blackwell GPU).
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import (
|
||||
compress_norm_rope_store_cpu,
|
||||
compress_norm_rope_store_indexer_cpu,
|
||||
save_partial_states_cpu,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.compressor import CompressorMetadata, DeepseekCompressor
|
||||
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
|
||||
|
||||
|
||||
class DeepseekV4CPUCompressor(DeepseekCompressor):
|
||||
"""CPU compressor: same state/weights as the shared base, but
|
||||
``forward``'s cache-write and compress->RMSNorm->RoPE->quant->store
|
||||
steps always dispatch straight to the ported CPU kernels above instead
|
||||
of through the shared method's platform-dispatch chain. This class
|
||||
only ever runs on CPU, and ``use_fp4_cache`` is always ``False`` here
|
||||
(the indexer's MXFP4 cache requires a Blackwell GPU), so the shared
|
||||
method's MXFP4/two-stage/cutedsl/triton branches never apply.
|
||||
"""
|
||||
|
||||
_norm_weight_fp32: torch.Tensor
|
||||
|
||||
def cache_norm_weight_fp32(self) -> None:
|
||||
"""Cache a fp32 contiguous copy of ``norm.weight`` once, right after
|
||||
weight loading (see
|
||||
``DeepseekV4CPUAttention.process_weights_after_loading``) -- the CPU
|
||||
kernel requires fp32/contiguous but the checkpoint loads this weight
|
||||
in bf16, and it never changes after loading."""
|
||||
self._norm_weight_fp32 = self.norm.weight.to(torch.float32).contiguous()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
kv_score: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb,
|
||||
) -> None:
|
||||
kv, score = kv_score.split(
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
|
||||
)
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
|
||||
state_metadata = attn_metadata[self.state_cache.prefix]
|
||||
assert isinstance(state_metadata, CompressorMetadata)
|
||||
token_to_req_indices = state_metadata.token_to_req_indices
|
||||
assert token_to_req_indices is not None
|
||||
slot_mapping = state_metadata.slot_mapping
|
||||
num_actual = slot_mapping.shape[0]
|
||||
block_table = state_metadata.block_table
|
||||
block_size = state_metadata.block_size
|
||||
|
||||
state_cache = self.state_cache.kv_cache
|
||||
|
||||
save_partial_states_cpu(
|
||||
kv, score, self.ape, positions, state_cache, slot_mapping
|
||||
)
|
||||
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
assert self.head_dim in (512, 128), (
|
||||
f"unsupported compressor head_dim: {self.head_dim}"
|
||||
)
|
||||
window = (1 + self.overlap) * self.compress_ratio
|
||||
win_offsets = torch.arange(window, device=positions.device)
|
||||
win_local = positions.unsqueeze(-1) - window + 1 + win_offsets
|
||||
req_idx = token_to_req_indices[:num_actual]
|
||||
gather_slots = map_local_to_global_slots_cpu(
|
||||
win_local, req_idx, block_table, block_size
|
||||
)
|
||||
|
||||
kv_cache_2d = kv_cache.view(kv_cache.shape[0], -1)
|
||||
kv_cache_block_size = kv_cache.shape[1]
|
||||
|
||||
store_fn = (
|
||||
compress_norm_rope_store_cpu
|
||||
if self.head_dim == 512
|
||||
else compress_norm_rope_store_indexer_cpu
|
||||
)
|
||||
store_fn(
|
||||
state_cache,
|
||||
gather_slots,
|
||||
positions,
|
||||
k_cache_metadata.slot_mapping,
|
||||
self._norm_weight_fp32,
|
||||
self.rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
kv_cache_2d,
|
||||
kv_cache_block_size,
|
||||
self.compress_ratio,
|
||||
)
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 sparse-MLA backend descriptor.
|
||||
|
||||
DeepSeek-V4 attention runs entirely through
|
||||
``DeepseekV4Attention``/``DeepseekV4CPUAttention``, never through a generic
|
||||
``AttentionImpl``. ``DeepseekV4CPUSparseBackend`` exists so the CPU attention
|
||||
layer has its own backend name and its own metadata-builder class: CPU is not
|
||||
allowed to run triton-cpu inside the model's forward path, so
|
||||
``DeepseekV4CPUFlashMLAMetadataBuilder`` overrides the shared base's
|
||||
``_build_c128a_metadata`` (triton-backed in
|
||||
``DeepseekV4SparseMLAMetadataBuilder``) with a plain-eager-PyTorch
|
||||
reimplementation of the same per-token block-table-resolution math.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend,
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_LAYER_TYPE_C4A,
|
||||
_LAYER_TYPE_C128A,
|
||||
_LAYER_TYPE_SWAONLY,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta
|
||||
|
||||
|
||||
class DeepseekV4CPUFlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
"""CPU sparse-MLA metadata builder: same fields as the shared base, but
|
||||
skips the C128A dense-topk metadata entirely -- unlike CUDA/XPU/ROCm,
|
||||
``DeepseekV4CPUAttention.forward_mqa`` never reads
|
||||
``c128a_global_decode_topk_indices``/``c128a_decode_topk_lens``/
|
||||
``c128a_prefill_topk_indices``; it recomputes the same local top-k
|
||||
directly from ``positions`` instead, so building it here (whether
|
||||
eagerly or via the shared base's triton kernel) would be wasted work.
|
||||
"""
|
||||
|
||||
def _build_c128a_metadata(
|
||||
self,
|
||||
cm: CommonAttentionMetadata,
|
||||
req_id_per_token: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
return {}
|
||||
|
||||
|
||||
class DeepseekV4CPUSparseBackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_V4_MLA_SPARSE"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4CPUFlashMLAMetadataBuilder]:
|
||||
return DeepseekV4CPUFlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder):
|
||||
"""CPU indexer metadata builder: same fields as the shared base, but
|
||||
prefill requests are never split into multiple chunks.
|
||||
|
||||
``DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks`` (the
|
||||
shared base's default) bounds two things the CUDA/XPU indexer kernels
|
||||
need: the flat-gather workspace size and the dense M*N logits tensor
|
||||
those kernels allocate for a chunk. The CPU indexer
|
||||
(``sparse_attn_indexer_cpu``) allocates neither -- it reads the paged
|
||||
K-cache directly via ``fp8_paged_mqa_logits_cpu``/
|
||||
``topk_transform_512_cpu``, with no chunk-splitting of its own.
|
||||
With chunked prefill now enabled for this model on CPU (see
|
||||
``CpuPlatform.check_and_update_config``'s ``amx_mla_or_dsv4_enabled``),
|
||||
the total query-token count for one step is already bounded by
|
||||
``max_num_batched_tokens`` -- there is nothing left here to bound, so
|
||||
this always returns a single chunk spanning the whole step's prefill
|
||||
batch.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _split_indexer_prefill_chunks(
|
||||
compressed_seq_lens_cpu: torch.Tensor,
|
||||
prefill_query_lens_cpu: torch.Tensor,
|
||||
workspace_size: int,
|
||||
max_logits_bytes: int,
|
||||
request_offset: int = 0,
|
||||
) -> list[tuple[slice, slice]]:
|
||||
# workspace_size/max_logits_bytes are unused: no flat-gather
|
||||
# workspace or dense M*N buffer is allocated.
|
||||
num_reqs = compressed_seq_lens_cpu.shape[0]
|
||||
total_query_len = int(prefill_query_lens_cpu.sum().item())
|
||||
return [
|
||||
(
|
||||
slice(request_offset, request_offset + num_reqs),
|
||||
slice(0, total_query_len),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerBackend(DeepseekV4IndexerBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_V4_INDEXER"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4CPUIndexerMetadataBuilder]:
|
||||
return DeepseekV4CPUIndexerMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4CPUSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
"""CPU SWA metadata builder: same fields as the shared base."""
|
||||
|
||||
def build_tile_scheduler(
|
||||
self, num_decode_tokens: int
|
||||
) -> dict[str, FlashMLASchedMeta | None]:
|
||||
"""CPU never runs the FlashMLA tile-scheduler planner (that's a CUDA
|
||||
C++ decode-path concern) -- always return the all-``None`` sentinel
|
||||
the shared base's own CPU branch returns."""
|
||||
return {
|
||||
_LAYER_TYPE_SWAONLY: None,
|
||||
_LAYER_TYPE_C4A: None,
|
||||
_LAYER_TYPE_C128A: None,
|
||||
}
|
||||
@@ -0,0 +1,648 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 attention subclass.
|
||||
|
||||
``forward_mqa`` resolves SWA/compressed top-k indices to paged-cache slot
|
||||
ids and calls the fused ``flash_mla_with_kvcache_cpu`` kernel. ``_o_proj``
|
||||
stays eager -- not on the attention hot path.
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4Attention,
|
||||
DeepseekV4Indexer,
|
||||
DeepseekV4IndexerCache,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_compressor import DeepseekV4CPUCompressor
|
||||
from vllm.models.deepseek_v4.cpu.cpu_mla import (
|
||||
DeepseekV4CPUIndexerBackend,
|
||||
DeepseekV4CPUSparseBackend,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_utils import map_local_to_global_slots_cpu
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_ROPE_DIM = 64
|
||||
|
||||
|
||||
def _deepseek_v4_cpu_prepare_and_attn(
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor | None,
|
||||
indexer_kv_score: torch.Tensor | None,
|
||||
indexer_weights: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""Opaque custom-op boundary around attention-prep + sparse-indexer +
|
||||
MLA attention + output projection -- CPU-only.
|
||||
|
||||
Wraps cache writes with no explicit tensor args (SWA/compressor/indexer
|
||||
state), which torch.compile can't otherwise track as mutations -- one
|
||||
opaque call sidesteps that. ``_o_proj`` is folded in too because CPU's
|
||||
``DYNAMO_TRACE_ONCE`` mode traces exactly once, during warmup when
|
||||
``attn_metadata`` is ``None``; tracing ``_o_proj`` separately would bake
|
||||
that warmup branch into the compiled graph permanently, dead-code-
|
||||
eliminating the whole attention computation upstream.
|
||||
"""
|
||||
layer_name_str = _resolve_layer_name(layer_name)
|
||||
self = get_forward_context().no_compile_layers[layer_name_str]
|
||||
self._prepare_and_attn_fn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
|
||||
def _deepseek_v4_cpu_prepare_and_attn_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor | None,
|
||||
indexer_kv_score: torch.Tensor | None,
|
||||
indexer_weights: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
# hidden_states' last dim already equals _o_proj's output width, so we
|
||||
# don't need to resolve layer_name (an opaque object) here.
|
||||
return hidden_states.new_empty((hidden_states.shape[0], hidden_states.shape[-1]))
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="deepseek_v4_cpu_prepare_and_attn",
|
||||
op_func=_deepseek_v4_cpu_prepare_and_attn,
|
||||
mutates_args=["o_padded"],
|
||||
fake_impl=_deepseek_v4_cpu_prepare_and_attn_fake,
|
||||
)
|
||||
|
||||
|
||||
def _dequant_linear_weight(layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Dequantize a linear layer's weight, including FP8 block scales.
|
||||
|
||||
``_o_proj`` reads ``wo_a.weight`` directly (not via
|
||||
``quant_method.apply()``), so the per-block FP8 scale must be applied
|
||||
here explicitly.
|
||||
"""
|
||||
weight = layer.weight
|
||||
scale = getattr(layer, "weight_scale_inv", None)
|
||||
if scale is None:
|
||||
scale = getattr(layer, "weight_scale", None)
|
||||
if scale is None:
|
||||
return weight.float()
|
||||
|
||||
if scale.dtype == torch.float8_e8m0fnu:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
scale = _upcast_e8m0_to_fp32(scale)
|
||||
scale = scale.to(torch.float32)
|
||||
|
||||
if scale.numel() == 1:
|
||||
return weight.float() * scale
|
||||
|
||||
block_size = getattr(layer, "weight_block_size", None)
|
||||
if block_size is None:
|
||||
# Per-channel scale: one row-scale per output row.
|
||||
return weight.float() * scale.view(-1, 1)
|
||||
|
||||
block_m, block_k = block_size
|
||||
m, k = weight.shape
|
||||
scale = scale.repeat_interleave(block_m, dim=0).repeat_interleave(block_k, dim=1)
|
||||
return weight.float() * scale[:m, :k]
|
||||
|
||||
|
||||
def _fused_indexer_q_rope_quant_cpu(
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""CPU-only equivalent of ``fused_indexer_q_rope_quant``'s FP8 path
|
||||
(CPU never reaches its MXFP4 arm -- Blackwell-only). Returns
|
||||
``(q_fp8, weights_out)``, folding the per-token q scale into
|
||||
``index_weights``. ``positions``/``index_q_cos_sin_cache`` are already
|
||||
contiguous int64/fp32, so no conversion is needed here.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert index_q.ndim == 3
|
||||
assert index_q_cos_sin_cache.ndim == 2
|
||||
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=current_platform.fp8_dtype())
|
||||
ops.fused_indexer_q_rope_quant_cpu(
|
||||
positions,
|
||||
index_q.to(torch.float32),
|
||||
index_q_cos_sin_cache,
|
||||
index_q_fp8,
|
||||
index_weights.to(torch.float32),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out,
|
||||
)
|
||||
return index_q_fp8, index_weights_out
|
||||
|
||||
|
||||
class _NoOpEvent:
|
||||
"""Stand-in for ``torch.cuda.Event`` on CPU: since CPU never sets an aux
|
||||
stream list, these events are constructed but never actually used."""
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexerCache(DeepseekV4IndexerCache):
|
||||
"""CPU indexer K-cache descriptor: same fields as the shared base, just
|
||||
pointing ``get_attn_backend()`` at ``DeepseekV4CPUIndexerBackend``
|
||||
instead of the shared, CUDA/XPU-oriented ``DeepseekV4IndexerBackend``."""
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV4CPUIndexerBackend
|
||||
|
||||
|
||||
class DeepseekV4CPUIndexer(DeepseekV4Indexer):
|
||||
"""CPU indexer: the C4A short-context fallback runs as eager PyTorch
|
||||
instead of Triton. Never constructed directly -- ``__class__`` is
|
||||
swapped onto an existing ``DeepseekV4Indexer`` instance post-construction.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
compressed_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
qr_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""CPU override: no aux streams, so wq_b_and_q_quant and the
|
||||
compressor run straight-line instead of through
|
||||
``maybe_execute_in_parallel``; the short-context check drops the
|
||||
CUDA-only ``is_current_stream_capturing()`` guard (always False
|
||||
here)."""
|
||||
compressor = self.compressor
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
|
||||
if indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens:
|
||||
# Fewer candidates than topk: all are selected, but the K
|
||||
# cache still needs to be built.
|
||||
compressor(compressed_kv_score, positions, rotary_emb)
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_tokens = (
|
||||
indexer_metadata.num_decode_tokens
|
||||
+ indexer_metadata.num_prefill_tokens
|
||||
)
|
||||
if num_tokens > 0:
|
||||
self._fill_short_context_topk(positions, num_tokens)
|
||||
return None, None, None
|
||||
|
||||
def wq_b_and_q_quant():
|
||||
q = self._wq_b_proj(qr, qr_scale)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
return _fused_indexer_q_rope_quant_cpu(
|
||||
positions,
|
||||
q,
|
||||
rotary_emb.cos_sin_cache,
|
||||
indexer_weights,
|
||||
self.softmax_scale,
|
||||
self.n_head**-0.5,
|
||||
)
|
||||
|
||||
# Order matches the base class's parallel-join contract: compressor
|
||||
# must complete before indexer_op runs (skip_k_cache_insert=True).
|
||||
q, weights = wq_b_and_q_quant()
|
||||
compressor(compressed_kv_score, positions, rotary_emb)
|
||||
# No q_scale: FP8-only path (MXFP4 needs Blackwell).
|
||||
return q, None, weights
|
||||
|
||||
def _fill_short_context_topk(
|
||||
self, positions: torch.Tensor, num_tokens: int
|
||||
) -> None:
|
||||
assert self.topk_indices_buffer is not None
|
||||
device = positions.device
|
||||
arange = torch.arange(self.topk_tokens, device=device)
|
||||
num_compressed = torch.div(
|
||||
positions[:num_tokens] + 1, self.compress_ratio, rounding_mode="floor"
|
||||
)
|
||||
valid = arange.unsqueeze(0) < num_compressed.unsqueeze(-1)
|
||||
filled = torch.where(
|
||||
valid, arange.unsqueeze(0), torch.full_like(arange.unsqueeze(0), -1)
|
||||
)
|
||||
self.topk_indices_buffer[:num_tokens, : self.topk_tokens] = filled.to(
|
||||
self.topk_indices_buffer.dtype
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4CPUAttention(DeepseekV4Attention):
|
||||
"""CPU sparse MLA attention layer for DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4CPUSparseBackend
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# Swap classes post-construction (no class-swap hook on the shared
|
||||
# base) so forward()/get_attn_backend() dispatch to CPU kernels.
|
||||
# SWA-only layers (compress_ratio <= 1) never build a
|
||||
# compressor/indexer.
|
||||
if self.compressor is not None:
|
||||
self.compressor.__class__ = DeepseekV4CPUCompressor
|
||||
if self.indexer is not None:
|
||||
self.indexer.__class__ = DeepseekV4CPUIndexer
|
||||
self.indexer.k_cache.__class__ = DeepseekV4CPUIndexerCache
|
||||
self.indexer.compressor.__class__ = DeepseekV4CPUCompressor
|
||||
|
||||
self._wo_a_packed: torch.Tensor | None = None
|
||||
self._wrap_wo_a_process_weights_after_loading()
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
"""Cache fp32-contiguous copies of tensors CPU kernels want that way
|
||||
but that arrive bf16 (rotary cos/sin table, compressor RMSNorm
|
||||
weights) -- cast once here instead of per forward call.
|
||||
|
||||
Runs after every quantized layer's own
|
||||
``process_weights_after_loading`` (see ``is_deferred_attention_layer``);
|
||||
``wo_a``'s packing must happen *before* that phase instead, so it's a
|
||||
separate monkeypatch in ``__init__``.
|
||||
"""
|
||||
self.rotary_emb.cos_sin_cache = self.rotary_emb.cos_sin_cache.to(
|
||||
torch.float32
|
||||
).contiguous()
|
||||
if self.compressor is not None:
|
||||
self.compressor.cache_norm_weight_fp32()
|
||||
if self.indexer is not None:
|
||||
self.indexer.compressor.cache_norm_weight_fp32()
|
||||
|
||||
def _wrap_wo_a_process_weights_after_loading(self) -> None:
|
||||
"""Snapshot and pack ``wo_a``'s weight into a bf16 copy for
|
||||
``bmm_cpu`` before the FP8 kernel's own
|
||||
``process_weights_after_loading`` VNNI-repacks it in place for
|
||||
row-major reads ``_o_proj`` never does."""
|
||||
wo_a = self.wo_a
|
||||
quant_method = wo_a.quant_method
|
||||
orig_pwal = quant_method.process_weights_after_loading
|
||||
heads_per_group = self.n_local_heads // self.n_local_groups
|
||||
k_dim = heads_per_group * self.head_dim
|
||||
|
||||
def _capture_and_pack(layer: torch.nn.Module) -> None:
|
||||
dequant = _dequant_linear_weight(layer).to(torch.bfloat16)
|
||||
dequant = dequant.view(self.n_local_groups, self.o_lora_rank, k_dim)
|
||||
self._wo_a_packed = torch.ops._C.convert_weight_packed(dequant.contiguous())
|
||||
orig_pwal(layer)
|
||||
# orig_pwal VNNI-repacks layer.weight/scale for ordinary-linear
|
||||
# use that _o_proj never needs -- free them, same pattern as
|
||||
# dispatch_cpu_unquantized_gemm's remove_weight=True.
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
scale_attr = (
|
||||
"weight_scale_inv"
|
||||
if hasattr(layer, "weight_scale_inv")
|
||||
else "weight_scale"
|
||||
)
|
||||
if hasattr(layer, scale_attr):
|
||||
setattr(
|
||||
layer,
|
||||
scale_attr,
|
||||
torch.nn.Parameter(torch.empty(0), requires_grad=False),
|
||||
)
|
||||
|
||||
quant_method.process_weights_after_loading = ( # type: ignore[method-assign]
|
||||
_capture_and_pack
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: wraps attention-prep + MLA + output projection in the
|
||||
CPU-only opaque custom op (see ``_deepseek_v4_cpu_prepare_and_attn``).
|
||||
"""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
o_padded = torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
|
||||
|
||||
kv_score: torch.Tensor | None = None
|
||||
if self.compressor is not None:
|
||||
# Must go through the module's own forward (not a raw matmul on
|
||||
# its weight): the packed-GEMM kernel frees the raw weight after
|
||||
# packing, same as wo_a's.
|
||||
kv_score = self.compressor.fused_wkv_wgate(hidden_states).to(torch.float32)
|
||||
|
||||
indexer_weights: torch.Tensor | None = None
|
||||
indexer_kv_score: torch.Tensor | None = None
|
||||
if self.indexer is not None:
|
||||
indexer_weights, _ = self.indexer.weights_proj(hidden_states)
|
||||
indexer_kv_score = self.indexer.compressor.fused_wkv_wgate(
|
||||
hidden_states
|
||||
).to(torch.float32)
|
||||
|
||||
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
||||
|
||||
return torch.ops.vllm.deepseek_v4_cpu_prepare_and_attn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def _fused_qnorm_rope_kv_insert(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
attn_metadata: (
|
||||
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
|
||||
),
|
||||
) -> torch.Tensor:
|
||||
"""CPU override: only the fp8_ds_mla (uint8) SWA cache layout is
|
||||
supported here, so the base method's bf16/per-tensor-fp8 branches
|
||||
are dropped."""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
return q
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
# positions is already int64 (the runner's buffer), no cast needed.
|
||||
assert positions.dtype == torch.int64
|
||||
assert swa_kv_cache.dtype == torch.uint8, (
|
||||
"DeepseekV4CPUAttention only supports the fp8_ds_mla cache layout"
|
||||
)
|
||||
# Fused: Q side does per-head weight-free RMSNorm + GPT-J RoPE
|
||||
# (zero-filling padded head slots); KV side does GPT-J RoPE + UE8M0
|
||||
# FP8 quant + paged insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
return ops.fused_qnorm_rope_kv_insert_cpu(
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.padded_heads,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""CPU override: two ``RMSNorm`` calls instead of the shared
|
||||
``fused_q_kv_rmsnorm``, whose raw ``@triton.jit`` kernel fails to
|
||||
link under triton-cpu (``undefined symbol: __truncdfbf2``)."""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
return self.q_norm(qr), None, self.kv_norm(kv)
|
||||
|
||||
def _prepare_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: no aux streams, so query-projection+KV-insert, the
|
||||
indexer, and the compressor run straight-line instead of through
|
||||
execute_in_parallel/maybe_execute_in_parallel."""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
|
||||
q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_local_heads, self.head_dim)
|
||||
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
index_q: torch.Tensor | None = None
|
||||
index_q_scale: torch.Tensor | None = None
|
||||
index_weights_out: torch.Tensor | None = None
|
||||
if indexer is not None:
|
||||
assert compressor is not None
|
||||
index_q, index_q_scale, index_weights_out = indexer(
|
||||
hidden_states,
|
||||
qr,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
self.indexer_rotary_emb,
|
||||
qr_scale,
|
||||
)
|
||||
compressor(kv_score, positions, self.rotary_emb)
|
||||
elif compressor is not None:
|
||||
compressor(kv_score, positions, self.rotary_emb)
|
||||
|
||||
self._sparse_indexer_and_attn(
|
||||
hidden_states,
|
||||
index_q,
|
||||
index_q_scale,
|
||||
index_weights_out,
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
|
||||
def _sparse_indexer_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
index_q: torch.Tensor | None,
|
||||
index_q_scale: torch.Tensor | None,
|
||||
index_weights: torch.Tensor | None,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""CPU override: identical body, minus ``@eager_break_during_capture``
|
||||
(this platform never captures a CUDA graph, so it was a no-op)."""
|
||||
if self.indexer is not None and index_q is not None:
|
||||
assert index_weights is not None
|
||||
# index_q_scale is always None on CPU (FP8-only path).
|
||||
assert index_q_scale is None
|
||||
self.indexer.indexer_op(hidden_states, index_q, None, index_weights)
|
||||
|
||||
self.forward_mqa(q, kv, positions, out)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# No head-count padding constraint for the (not yet ported) CPU kernel.
|
||||
return num_heads
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
if get_forward_context().attn_metadata is None:
|
||||
# Warmup/profile run: no real metadata, o is a zeroed placeholder.
|
||||
return torch.zeros(
|
||||
(o.shape[0], self.hidden_size), dtype=o.dtype, device=o.device
|
||||
)
|
||||
|
||||
num_tokens, num_heads, _ = o.shape
|
||||
cos_sin_cache = self.rotary_emb.cos_sin_cache
|
||||
# Undo the query's own GPT-J RoPE rotation: the attention output is
|
||||
# a weighted sum of KV rows at different positions, so R(-pos_q)
|
||||
# recovers the translation-invariant quantity wo_a/wo_b expect.
|
||||
o_derot = ops.inverse_gptj_rope_o_proj_cpu(
|
||||
o, positions, cos_sin_cache, _ROPE_DIM
|
||||
)
|
||||
|
||||
heads_per_group = num_heads // self.n_local_groups
|
||||
o_grouped = o_derot.reshape(
|
||||
num_tokens, self.n_local_groups, heads_per_group * self.head_dim
|
||||
).to(torch.bfloat16)
|
||||
assert self._wo_a_packed is not None
|
||||
z = o_grouped.new_empty((self.n_local_groups, num_tokens, self.o_lora_rank))
|
||||
ops.bmm_cpu(
|
||||
z,
|
||||
o_grouped.transpose(0, 1).contiguous(),
|
||||
self._wo_a_packed,
|
||||
True,
|
||||
None,
|
||||
)
|
||||
z = z.transpose(0, 1).to(o.dtype)
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup/profile run: reserve nothing extra, skip real kernels.
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
swa_metadata = cast(
|
||||
DeepseekSparseSWAMetadata,
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
assert swa_metadata.token_to_req_indices is not None
|
||||
req_idx = swa_metadata.token_to_req_indices
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
|
||||
num_tokens = q.shape[0]
|
||||
device = q.device
|
||||
|
||||
# ---- SWA window: causal sliding window over the raw per-token cache
|
||||
window = self.window_size
|
||||
win_offsets = torch.arange(window, device=device)
|
||||
start_pos = (positions - window + 1).clamp(min=0)
|
||||
local_pos = start_pos.unsqueeze(-1) + win_offsets
|
||||
valid_win = local_pos <= positions.unsqueeze(-1)
|
||||
win_local = torch.where(valid_win, local_pos, torch.full_like(local_pos, -1))
|
||||
window_slots = map_local_to_global_slots_cpu(
|
||||
win_local, req_idx, swa_metadata.block_table, swa_metadata.block_size
|
||||
)
|
||||
window_cache_2d = self.swa_cache_layer.kv_cache.view(
|
||||
self.swa_cache_layer.kv_cache.shape[0], -1
|
||||
)
|
||||
|
||||
# ---- Compressed KV: C4A's learned top-k, or C128A's dense top-k ----
|
||||
if swa_only:
|
||||
compressed_slots = window_slots.new_full((num_tokens, 0), -1)
|
||||
# Unused when num_compressed == 0 (only dtype/2D-ness matter);
|
||||
# reuse the window cache to avoid an allocation.
|
||||
compressed_cache_2d = window_cache_2d
|
||||
compressed_block_size = 1
|
||||
else:
|
||||
assert flashmla_metadata is not None
|
||||
main_block_size = flashmla_metadata.block_size // self.compress_ratio
|
||||
if self.compress_ratio == 4:
|
||||
assert self.topk_indices_buffer is not None
|
||||
local_topk = self.topk_indices_buffer[:num_tokens]
|
||||
else:
|
||||
width = -(-self.max_model_len // self.compress_ratio)
|
||||
num_compressed = torch.div(
|
||||
positions + 1, self.compress_ratio, rounding_mode="floor"
|
||||
).clamp(max=width)
|
||||
arange_w = torch.arange(width, device=device)
|
||||
local_topk = torch.where(
|
||||
arange_w.unsqueeze(0) < num_compressed.unsqueeze(-1),
|
||||
arange_w.unsqueeze(0).expand(num_tokens, width),
|
||||
torch.full(
|
||||
(num_tokens, width), -1, device=device, dtype=torch.int64
|
||||
),
|
||||
)
|
||||
compressed_slots = map_local_to_global_slots_cpu(
|
||||
local_topk, req_idx, flashmla_metadata.block_table, main_block_size
|
||||
)
|
||||
compressed_cache_2d = self.kv_cache.view(self.kv_cache.shape[0], -1)
|
||||
compressed_block_size = main_block_size
|
||||
|
||||
torch.ops._C.flash_mla_with_kvcache_cpu(
|
||||
output,
|
||||
q,
|
||||
window_cache_2d,
|
||||
window_slots,
|
||||
swa_metadata.block_size,
|
||||
compressed_cache_2d,
|
||||
compressed_slots,
|
||||
compressed_block_size,
|
||||
self.attn_sink,
|
||||
self.scale,
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for DeepSeek-V4's CPU-ported kernels."""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def map_local_to_global_slots_cpu(
|
||||
local_indices: torch.Tensor,
|
||||
req_idx: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map per-request-local (token or compressed-token) indices to global
|
||||
paged-cache slot ids via a block table, mirroring
|
||||
``compute_global_topk_indices_and_lens``.
|
||||
|
||||
Args:
|
||||
local_indices: [N, K], -1 sentinel for invalid entries.
|
||||
req_idx: [N], row into ``block_table`` for each of the N rows.
|
||||
block_table: [num_reqs, max_blocks_per_seq].
|
||||
block_size: tokens (or compressed tokens) per physical block.
|
||||
|
||||
Returns:
|
||||
[N, K] int64 global slot ids, -1 where ``local_indices`` was invalid.
|
||||
"""
|
||||
valid = local_indices >= 0
|
||||
safe_local = local_indices.clamp(min=0).to(torch.int64)
|
||||
block_pos = torch.div(safe_local, block_size, rounding_mode="floor")
|
||||
offset = safe_local % block_size
|
||||
req_expand = req_idx.to(torch.int64).unsqueeze(-1).expand_as(local_indices)
|
||||
block_num = block_table[req_expand, block_pos].to(torch.int64)
|
||||
slot = block_num * block_size + offset
|
||||
return torch.where(valid, slot, torch.full_like(slot, -1))
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 DSpark (speculative decoding) — deferred, not implemented.
|
||||
|
||||
``DSparkDeepseekV4ForCausalLM`` is only instantiated when the speculative-
|
||||
decoding registry resolves it by name for a DSpark speculative config; a
|
||||
plain (non-spec) DeepSeek-V4 CPU model never constructs this class. This
|
||||
stub exists solely so ``vllm.models.deepseek_v4.cpu`` is import-clean.
|
||||
"""
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM:
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 speculative decoding (DSpark) is not implemented on CPU."
|
||||
)
|
||||
@@ -0,0 +1,922 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_TILELANG_MHC,
|
||||
HCHeadOp,
|
||||
MHCFusedPostPreOp,
|
||||
MHCPostOp,
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.cpu.cpu_sparse import DeepseekV4CPUAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
swiglu_limit: float | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
is_sequence_parallel: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# is_sequence_parallel: weights are replicated, no collective ops
|
||||
# needed. Otherwise: standard TP with an allreduce at the end.
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
if swiglu_limit is not None:
|
||||
self.act_fn = SiluAndMulWithClamp(swiglu_limit)
|
||||
else:
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.prefix = prefix
|
||||
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.n_routed_experts = config.n_routed_experts
|
||||
self.n_activated_experts = config.num_experts_per_tok
|
||||
self.moe_intermediate_size = config.moe_intermediate_size
|
||||
self.swiglu_limit = config.swiglu_limit
|
||||
self.renormalize = config.norm_topk_prob
|
||||
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
|
||||
|
||||
self.gate = GateLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.n_routed_experts,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
# Use randint instead of empty to avoid garbage values causing
|
||||
# invalid memory access in dummy mode (--load-format="dummy")
|
||||
self.gate.tid2eid = nn.Parameter(
|
||||
torch.randint(
|
||||
0,
|
||||
config.n_routed_experts,
|
||||
(config.vocab_size, config.num_experts_per_tok),
|
||||
dtype=self.hash_indices_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
elif getattr(config, "topk_method", None) == "noaux_tc":
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
|
||||
# No CPU fused-shared-expert grouped-GEMM path yet.
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
|
||||
self.shared_experts = DeepseekV4MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
assert config.n_routed_experts % self.tp_size == 0
|
||||
|
||||
self.n_local_experts = config.n_routed_experts // self.tp_size
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
if is_hash_moe:
|
||||
# apply_monolithic() never sees hash_indices_table via the
|
||||
# router path (modular-only) -- stash it on RoutedExperts
|
||||
# directly. object.__setattr__ avoids nn.Module re-registering
|
||||
# (and duplicating in the state_dict) this alias.
|
||||
object.__setattr__(
|
||||
self.experts.routed_experts, "hash_indices_table", self.gate.tid2eid
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
if self.gate.tid2eid is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
|
||||
class DeepseekV4DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Lazy import to avoid top-level tilelang dependency.
|
||||
# Registers both torch.ops.vllm.mhc_pre and mhc_post
|
||||
import vllm.model_executor.layers.mhc # noqa: F401
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.attn = DeepseekV4CPUAttention(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_post_alpha = 2.0
|
||||
mix_hc = (2 + self.hc_mult) * self.hc_mult
|
||||
hc_dim = self.hc_mult * self.hidden_size
|
||||
self.hc_attn_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
(mix_hc, hc_dim),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
(mix_hc, hc_dim),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_attn_base = nn.Parameter(
|
||||
torch.empty(
|
||||
mix_hc,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_base = nn.Parameter(
|
||||
torch.empty(
|
||||
mix_hc,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_attn_scale = nn.Parameter(
|
||||
torch.empty(
|
||||
3,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_ffn_scale = nn.Parameter(
|
||||
torch.empty(
|
||||
3,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.mhc_pre = MHCPreOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
self.mhc_fused_post_pre = MHCFusedPostPreOp()
|
||||
self.use_fused_mhc = HAS_TILELANG_MHC and not (
|
||||
HAS_AITER_MHC and self.hidden_size % 256 == 0
|
||||
)
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
):
|
||||
post_mix, res_mix, layer_input = self.mhc_pre(
|
||||
residual=x,
|
||||
fn=hc_fn,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
)
|
||||
return layer_input, post_mix, res_mix
|
||||
|
||||
def hc_post(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
):
|
||||
return self.mhc_post(x, residual, post, comb)
|
||||
|
||||
def _forward_fused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
# Run standalone hc_pre on first layer
|
||||
residual = x
|
||||
x, post_mix, res_mix = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
else:
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
def _forward_unfused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x, None, None, None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None = None,
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
if not self.use_fused_mhc:
|
||||
return self._forward_unfused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
return self._forward_fused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
}
|
||||
)
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
|
||||
self.device = current_platform.device_type
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV4MoE,
|
||||
"ffn",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(
|
||||
self.hc_mult,
|
||||
self.hc_dim,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(
|
||||
self.hc_mult,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_op = HCHeadOp()
|
||||
spec_config = vllm_config.speculative_config
|
||||
needs_mtp_hidden_states = spec_config is not None and (
|
||||
spec_config.use_eagle() or spec_config.uses_draft_model()
|
||||
)
|
||||
if get_pp_group().is_last_rank and needs_mtp_hidden_states:
|
||||
self._mtp_hidden_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.hc_dim,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
self._mtp_hidden_buffer = None
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> IntermediateTensors:
|
||||
# PP intermediate tensors keep the (num_tokens, hc_mult, hidden_size)
|
||||
# multi-stream shape until hc_head() collapses it.
|
||||
return IntermediateTensors(
|
||||
{
|
||||
"hidden_states": torch.zeros(
|
||||
(batch_size, self.hc_mult, self.config.hidden_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
hidden_states = hidden_states.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
# EAGLE3/DSpark/DFlash aux hidden states: post-mhc hidden state at
|
||||
# each target layer, averaged over hc_mult streams to [T,
|
||||
# hidden_size]. Empty unless a draft model set aux_hidden_state_layers.
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
# Reused below to avoid computing hc_post twice if the last layer is
|
||||
# also an aux layer.
|
||||
final_aux_recon: torch.Tensor | None = None
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
if (idx + 1) in self.aux_hidden_state_layers:
|
||||
# Unfused path already applied hc_post; fused path needs it
|
||||
# applied here before averaging.
|
||||
if layer.use_fused_mhc:
|
||||
aux_recon = layer.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
final_aux_recon = aux_recon
|
||||
else:
|
||||
aux_recon = hidden_states
|
||||
aux_hidden_states.append(aux_recon.mean(dim=1))
|
||||
if layer is not None and layer.use_fused_mhc:
|
||||
# Reuse the last layer's hc_post output if already computed above.
|
||||
if (
|
||||
final_aux_recon is not None
|
||||
and self.end_layer in self.aux_hidden_state_layers
|
||||
):
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = layer.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
self.hc_head_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
("compressor.fused_wkv_wgate", "compressor.wkv", 0),
|
||||
("compressor.fused_wkv_wgate", "compressor.wgate", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_param_name(name: str) -> str:
|
||||
inv_name = f"{name}_inv"
|
||||
if name not in params_dict and inv_name in params_dict:
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
# Each MoE's own per-layer fusion decision is the source of truth,
|
||||
# so the redirect below can't diverge from how it was built.
|
||||
fuse_by_layer = {
|
||||
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
|
||||
for mod_name, mod in self.named_modules()
|
||||
if isinstance(mod, DeepseekV4MoE)
|
||||
}
|
||||
n_routed = self.config.n_routed_experts
|
||||
# Maps the single shared-expert tensor group to one appended slot;
|
||||
# multiple shared experts would need per-expert slicing (see
|
||||
# deepseek_v2.py). DeepSeek-V4 has n_shared_experts == 1.
|
||||
if any(fuse_by_layer.values()) and self.config.n_shared_experts != 1:
|
||||
raise NotImplementedError(
|
||||
"deepseek-v4 fused shared-expert loading supports only "
|
||||
f"n_shared_experts == 1, got {self.config.n_shared_experts}"
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Shared-expert fusion: redirect shared_experts.w{1,2,3} into
|
||||
# the appended routed-expert slot so it loads through the
|
||||
# routed expert (grouped GEMM) loader.
|
||||
if ".ffn.shared_experts.w" in name and fuse_by_layer.get(
|
||||
extract_layer_index(name), False
|
||||
):
|
||||
name = name.replace(
|
||||
".ffn.shared_experts.w",
|
||||
f".ffn.experts.{n_routed}.w",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
break
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(param_name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# E8M0 scales are float8_e8m0fnu in checkpoints but the
|
||||
# MoE param is uint8; copy_() would numerically convert
|
||||
# (e.g. 2^-7 -> 0) instead of preserving raw bytes.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# return_success lets us skip experts that have
|
||||
# another replica available.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(param_name)
|
||||
continue
|
||||
|
||||
return loaded_params
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# (param_name, weight_name, expert_id, shard_id). When fusing shared
|
||||
# experts, include the appended slots so redirected shared-expert
|
||||
# weights route through the expert loader.
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.n_routed_experts + (
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
|
||||
def _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype: str, fuse_shared_experts: bool = False
|
||||
) -> WeightsMapper:
|
||||
if expert_dtype == "fp4":
|
||||
# MXFP4 experts register scales as ``w{1,2,3}_weight_scale`` (no
|
||||
# _inv); FP8 linear/shared-expert scales register as
|
||||
# ``weight_scale_inv``. Route DeepSeek-native ``.scale`` and
|
||||
# AMD-Quark ``.weight_scale`` keys accordingly, leaving expert/
|
||||
# shared-expert ``w{1,2,3}.weight_scale`` untouched for the MXFP4
|
||||
# loader.
|
||||
scale_regex = {
|
||||
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
|
||||
}
|
||||
else:
|
||||
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
|
||||
# scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys
|
||||
# there.
|
||||
scale_regex = {
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
}
|
||||
# Fused shared experts are redirected to routed expert slots elsewhere;
|
||||
# leave their names untouched here.
|
||||
substr_map: dict[str, str | None] = (
|
||||
{}
|
||||
if fuse_shared_experts
|
||||
else {".shared_experts.w2": ".shared_experts.down_proj"}
|
||||
)
|
||||
# No MTP support on CPU yet: drop mtp.* weights (no model.mtp submodule).
|
||||
substr_map["mtp."] = None
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"layers.": "model.layers.",
|
||||
"embed.": "model.embed.",
|
||||
"norm.": "model.norm.",
|
||||
"hc_head": "model.hc_head",
|
||||
"mtp.": "model.mtp.",
|
||||
},
|
||||
orig_to_new_regex=scale_regex,
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
".input_scale": ".input_scale_2",
|
||||
},
|
||||
orig_to_new_substr=substr_map,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
model_cls = DeepseekV4Model
|
||||
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype,
|
||||
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer (max_num_batched_tokens,
|
||||
hc_mult * hidden_size) for the MTP draft model. Populated by
|
||||
forward(); valid after each target step."""
|
||||
return getattr(self.model, "_mtp_hidden_buffer", None)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# No CPU AMX weight-packing hook yet.
|
||||
pass
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
@@ -0,0 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU DeepSeek-V4 MTP (speculative decoding) — deferred, not implemented.
|
||||
|
||||
``DeepSeekV4MTP`` is only instantiated when
|
||||
``vllm/model_executor/models/registry.py``'s speculative-decoding registry
|
||||
resolves it by name for a speculative-decoding config; a plain (non-spec)
|
||||
DeepSeek-V4 CPU model never constructs this class. This stub exists solely
|
||||
so ``vllm.models.deepseek_v4.cpu`` is import-clean.
|
||||
"""
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class DeepSeekV4MTP:
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 speculative decoding (MTP) is not implemented on CPU."
|
||||
)
|
||||
Reference in New Issue
Block a user