[Feat] DeepSeek V4 Rebased (#40860)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: qizixi <zixi@inferact.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: Simon Mo <simon@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Zhewen Li <jerven.vllm@gmail.com> Co-authored-by: Zijing Liu <liuzijing2014@gmail.com> Co-authored-by: khluu <khluu000@gmail.com> Co-authored-by: qizixi <zixi@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
co-authored by
Yongye Zhu
Yongye Zhu
Simon Mo
Bugen Zhao
Giancarlo Delfin
Jee Jee Li
Nick Hill
Roger Wang
Roy Wang
Woosuk Kwon
youkaichao
Zhewen Li
Zijing Liu
khluu
qizixi
Zhewen Li
parent
32e45636e3
commit
4d51588e23
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import IntEnum
|
||||
from math import prod
|
||||
@@ -13,11 +14,11 @@ import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -95,6 +96,10 @@ class KVCacheSpec:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
"""
|
||||
The maximum possible memory usage of this KV cache in bytes.
|
||||
@@ -269,6 +274,15 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
|
||||
|
||||
def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
|
||||
if spec.alignment is None:
|
||||
return
|
||||
actual_page_size = spec.real_page_size_bytes
|
||||
padded_page_size = round_up(actual_page_size, spec.alignment)
|
||||
if padded_page_size != actual_page_size:
|
||||
object.__setattr__(spec, "page_size_padded", padded_page_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class TQFullAttentionSpec(FullAttentionSpec):
|
||||
"""FullAttentionSpec with TQ-aware page size.
|
||||
@@ -299,15 +313,31 @@ class TQFullAttentionSpec(FullAttentionSpec):
|
||||
class MLAAttentionSpec(FullAttentionSpec):
|
||||
# TODO(Lucas/Chen): less hacky way to do this
|
||||
cache_dtype_str: str | None = None
|
||||
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
compress_ratio: int = 1 # Default to 1 for no compression.
|
||||
model_version: str | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.cache_dtype_str == "fp8_ds_mla":
|
||||
# See `vllm/v1/attention/backends/mla/flashmla_sparse.py`
|
||||
# for details.
|
||||
if self.model_version == "deepseek_v4":
|
||||
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
|
||||
# head_size stays semantic (512); bytes are determined here.
|
||||
return self.storage_block_size * 584
|
||||
# V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 +
|
||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
||||
return self.block_size * 656
|
||||
return (
|
||||
self.block_size
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* self.head_size
|
||||
* get_dtype_size(self.dtype)
|
||||
@@ -319,9 +349,15 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
"All attention layers in the same KV cache group must be MLAAttentionSpec."
|
||||
)
|
||||
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
|
||||
assert len(cache_dtype_str_set) == 1, (
|
||||
compress_ratio_set = set(spec.compress_ratio for spec in specs)
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(compress_ratio_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method."
|
||||
"quantization method, compress ratio, and model version."
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -331,6 +367,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
)
|
||||
|
||||
|
||||
@@ -393,6 +431,71 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
return (cdiv(num_tokens, self.block_size) + 1) * self.page_size_bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
"""Sliding window attention with MLA cache format."""
|
||||
|
||||
cache_dtype_str: str | None = None
|
||||
# DeepseekV4-only: see MLAAttentionSpec.model_version.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
compress_ratio: int = 1
|
||||
model_version: str | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.model_version == "deepseek_v4":
|
||||
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
|
||||
return self.storage_block_size * 584
|
||||
assert self.model_version is None, (
|
||||
f"Unsupported model version: {self.model_version}"
|
||||
)
|
||||
return (
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* self.head_size
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
|
||||
"All attention layers in the same KV cache group must be "
|
||||
"SlidingWindowMLASpec."
|
||||
)
|
||||
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
|
||||
compress_ratio_set = set(spec.compress_ratio for spec in specs)
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(compress_ratio_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(sliding_window_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, compress ratio, model version and sliding "
|
||||
"window size."
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
sliding_window=sliding_window_set.pop(),
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
@@ -527,7 +630,17 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
# Different block sizes, not uniform.
|
||||
return False
|
||||
one_spec = next(iter(kv_cache_specs.values()))
|
||||
if isinstance(one_spec, FullAttentionSpec):
|
||||
# NOTE: Check subclasses before parent classes since isinstance()
|
||||
# returns True for subclasses.
|
||||
if isinstance(one_spec, SlidingWindowMLASpec):
|
||||
# SlidingWindowMLASpec is uniform if all specs are SlidingWindowMLASpec
|
||||
# with the same sliding_window size.
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowMLASpec)
|
||||
and spec.sliding_window == one_spec.sliding_window
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, FullAttentionSpec):
|
||||
return all(
|
||||
isinstance(spec, FullAttentionSpec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
@@ -571,6 +684,21 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
else:
|
||||
return None
|
||||
|
||||
# NOTE: below util functions are only used by DeepseekV4 for now.
|
||||
def get_page_sizes(self) -> list[int]:
|
||||
return list(set(spec.page_size_bytes for spec in self.kv_cache_specs.values()))
|
||||
|
||||
def get_num_layer_tuples(self) -> int:
|
||||
return Counter(
|
||||
spec.page_size_bytes for spec in self.kv_cache_specs.values()
|
||||
).most_common(1)[0][1]
|
||||
|
||||
def max_memory_usage_pages(self, vllm_config: VllmConfig) -> int:
|
||||
return max(
|
||||
cdiv(spec.max_memory_usage_bytes(vllm_config), spec.page_size_bytes)
|
||||
for spec in self.kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVCacheTensor:
|
||||
@@ -593,6 +721,8 @@ class KVCacheGroupSpec:
|
||||
layer_names: list[str]
|
||||
# The KV cache spec of this manager layer
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
is_eagle_group: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user