[Model] Support DeepSeek-V4.1-Flash (#56214)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-11 02:11:19 -07:00
committed by GitHub
co-authored by Jee Jee Li andyluo7 OpenAI Codex zjy0516 Claude Opus 5
parent e6821ceac9
commit e77daef89e
49 changed files with 3996 additions and 138 deletions
+6 -4
View File
@@ -2143,13 +2143,15 @@ class rocm_aiter_ops:
if not current_platform.is_rocm():
return
from vllm.platforms.rocm import on_gfx11
from vllm.platforms.rocm import on_gfx11, on_gfx950
if on_gfx11() and not _OPS_REGISTERED:
# This op has self-contained Triton/C++ implementations on gfx11
# and gfx950. Only its optional top-k fast path comes from aiter.
if (on_gfx11() or on_gfx950()) and not _OPS_REGISTERED:
direct_register_custom_op(
op_name="rocm_aiter_sparse_attn_indexer",
op_func=rocm_aiter_sparse_attn_indexer,
mutates_args=["topk_indices_buffer"],
mutates_args=["topk_indices_buffer", "candidate_blocks"],
fake_impl=rocm_aiter_sparse_attn_indexer_fake,
dispatch_key=current_platform.dispatch_key,
)
@@ -2327,7 +2329,7 @@ class rocm_aiter_ops:
direct_register_custom_op(
op_name="rocm_aiter_sparse_attn_indexer",
op_func=rocm_aiter_sparse_attn_indexer,
mutates_args=["topk_indices_buffer"],
mutates_args=["topk_indices_buffer", "candidate_blocks"],
fake_impl=rocm_aiter_sparse_attn_indexer_fake,
dispatch_key=current_platform.dispatch_key,
)
+9 -7
View File
@@ -42,6 +42,7 @@ from vllm.forward_context import (
is_forward_context_available,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.utils.torch_utils import weak_ref_tensor, weak_ref_tensors
@@ -56,6 +57,12 @@ def is_breakable_cudagraph_enabled() -> bool:
F = TypeVar("F", bound=Callable[..., Any])
def _weak_ref_capture_arg(arg: Any) -> Any:
if isinstance(arg, QuantizedActivation):
return arg.weak_ref()
return weak_ref_tensor(arg)
def eager_break_during_capture(fn: F) -> F:
"""Decorator that turns a custom-op Python kernel into a "break point"
for the breakable cudagraph capture.
@@ -105,13 +112,8 @@ def eager_break_during_capture(fn: F) -> F:
# Weak-ref args: strong refs in the replay lambda pin cudagraph-pool
# slots across batch descriptors. cudagraph owns the slot, so the
# weak_ref is safe to deref on replay.
weak_args = tuple(
weak_ref_tensor(a) if isinstance(a, torch.Tensor) else a for a in args
)
weak_kwargs = {
k: weak_ref_tensor(v) if isinstance(v, torch.Tensor) else v
for k, v in kwargs.items()
}
weak_args = tuple(_weak_ref_capture_arg(a) for a in args)
weak_kwargs = {k: _weak_ref_capture_arg(v) for k, v in kwargs.items()}
return capture.add_eager(lambda: fn(*weak_args, **weak_kwargs))
return wrapper # type: ignore[return-value]
+19 -9
View File
@@ -15,6 +15,14 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# Architecture -> the hf_text_config field naming its n-gram layers. A model is
# only configurable here if it actually has such layers to store.
_NGRAM_LAYER_FIELDS = {
"DeepseekV41ForCausalLM": "engram_layer_ids",
"Qwen4ExpForCausalLM": "ple_layer_ids",
"Qwen4ExpForConditionalGeneration": "ple_layer_ids",
}
def _default_cpu_offload() -> bool:
"""Honor the legacy environment variable only when the field is omitted."""
@@ -40,23 +48,25 @@ class EngramConfig:
Otherwise, each DP rank has a separate TP-sharded embedding replica."""
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
"""Reject Engram configuration for models without supported embeddings."""
"""Reject Engram configuration for models without n-gram embeddings."""
from vllm.platforms import current_platform
supported_architectures = {
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
}
field = (
_NGRAM_LAYER_FIELDS.get(model_config.architecture)
if model_config is not None
else None
)
if (
model_config is None
or model_config.architecture not in supported_architectures
or field is None
or not current_platform.is_cuda()
or not getattr(model_config.hf_text_config, "ple_layer_ids", None)
or not getattr(model_config.hf_text_config, field, None)
):
raise ValueError(
"EngramConfig requires a model with supported Engram "
"embeddings. Currently only the CUDA Qwen4Exp implementation "
"with non-empty ple_layer_ids is supported."
"embeddings. Currently only the CUDA Qwen4Exp and DeepSeek "
"V4.1 implementations with non-empty n-gram layer ids are "
"supported."
)
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
+1
View File
@@ -165,6 +165,7 @@ FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
{
"DeepseekV4ForCausalLM",
"DeepSeekV4MTPModel",
"DeepseekV41ForCausalLM",
}
)
+40 -8
View File
@@ -679,12 +679,18 @@ class SpeculativeConfig:
],
}
)
if hf_config.model_type == "deepseek_v4":
if hf_config.model_type in ("deepseek_v4", "deepseek_v41"):
# V4.1 has no classic-MTP draft: its checkpoints ship DSpark stages
# under ``mtp.*``, so only V4 gets an MTP architecture here. The
# DSpark path rewrites ``architectures`` itself and needs only
# ``n_predict``; ``method="mtp"`` on V4.1 is rejected below.
is_v41 = hf_config.model_type == "deepseek_v41"
hf_config.model_type = "deepseek_mtp"
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
hf_config.update(
{"n_predict": n_predict, "architectures": ["DeepSeekV4MTPModel"]}
)
overrides = {"n_predict": n_predict}
if not is_v41:
overrides["architectures"] = ["DeepSeekV4MTPModel"]
hf_config.update(overrides)
if hf_config.model_type in ("pangu_ultra_moe"):
hf_config.model_type = "pangu_ultra_moe_mtp"
if hf_config.model_type == "pangu_ultra_moe_mtp":
@@ -1335,6 +1341,18 @@ class SpeculativeConfig:
MTPModelTypes
):
self.method = "mtp"
if (
self.target_model_config is not None
and self.target_model_config.hf_config.model_type
== "deepseek_v41"
):
raise ValueError(
"DeepSeek V4.1 has no classic-MTP draft: its "
"checkpoints ship DSpark stages under mtp.* "
"(main_proj/markov_head/confidence_head) and carry "
"no e_proj/h_proj/enorm/hnorm/hc_head weights. Use "
"speculative method 'dspark' instead of 'mtp'."
)
if (
self.num_speculative_tokens > 1
and self.draft_model_config.hf_config.model_type
@@ -1398,12 +1416,26 @@ class SpeculativeConfig:
and "Gemma4DSparkModel" not in self.draft_model_config.architectures
and "K3DSparkModel" not in self.draft_model_config.architectures
):
# DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config
# DeepSeek-V4(.1) DSpark reuses the full target config
# and its weights ship in the target checkpoint.
self.draft_model_config.hf_config.model_type = "deepseek_v4"
self.draft_model_config.hf_config.architectures = [
"DSparkDraftModel"
is_v41 = (
self.target_model_config.hf_config.model_type == "deepseek_v41"
)
draft_hf_config = self.draft_model_config.hf_config
draft_hf_config.model_type = (
"deepseek_v41" if is_v41 else "deepseek_v4"
)
draft_hf_config.architectures = [
"DSparkV41DraftModel" if is_v41 else "DSparkDraftModel"
]
if is_v41:
# hf_config_override set n_predict to the number of
# MTP stages (3), but one DSpark round drafts
# dspark_block_size tokens; num_speculative_tokens
# divisibility is checked against n_predict below.
draft_hf_config.n_predict = getattr(
draft_hf_config, "dspark_block_size", None
) or getattr(draft_hf_config, "n_predict", None)
self.draft_model_config.quantization = (
self.target_model_config.quantization
)
+1
View File
@@ -81,6 +81,7 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
"DeepseekV4ForCausalLM",
"DeepseekV4ForConditionalGeneration",
"DeepSeekV4MTPModel",
"DeepseekV41ForCausalLM",
"Dots3NoteForCausalLM",
"Dots3NoteMTPModel",
"Glm5NextForCausalLM",
@@ -91,6 +91,7 @@ class MooncakeStoreCoordinator:
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.mamba_group_ids = {
@@ -460,11 +460,13 @@ class NixlBaseConnectorWorker:
for spec in iter_layer_specs(group.kv_cache_spec)
)
]
if len(ple_groups) != 1 or len(ple_groups[0][1].layer_names) != 1:
if len(ple_groups) > 1 or any(
len(group.layer_names) != 1 for _, group in ple_groups
):
raise ValueError(
"CSA-linear NIXL requires exactly one PLE cache owner."
"CSA-linear NIXL requires at most one PLE cache owner."
)
self._ple_group_index = ple_groups[0][0]
self._ple_group_index = ple_groups[0][0] if ple_groups else None
if self._has_mamba:
assert self._is_hma_required
@@ -8,7 +8,7 @@ expose_input_quant_key; the kernel validates and reads the activation via
as_quantized_activation.
"""
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -34,6 +34,16 @@ class QuantizedActivation:
orig_shape: torch.Size
quant_key: QuantKey
def weak_ref(self) -> "QuantizedActivation":
"""Return a copy with non-owning tensor references for CUDA graph replay."""
from vllm.utils.torch_utils import weak_ref_tensor
return replace(
self,
data=weak_ref_tensor(self.data),
scale=weak_ref_tensor(self.scale),
)
def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
"""Advertise the kernel's pre-quantized input key on the layer, if any.
@@ -115,7 +115,20 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
# lazy import to avoid triggering `torch.compile` too early
from vllm.config.quantization import _ONLINE_SHORTHANDS
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
if current_platform.is_cuda() or current_platform.is_rocm():
# The v4.1 class is the v4 one extended to accept model_type
# "deepseek_v41" and its 32x32 MXFP8 linear layout. V4.1 has
# platform-specific implementations for both CUDA and ROCm.
from vllm.models.deepseek_v4_1 import (
DeepseekV4FP8Config as DeepseekV41FP8Config,
)
deepseek_config: type[QuantizationConfig] = DeepseekV41FP8Config
else:
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
deepseek_config = DeepseekV4FP8Config
from .auto_awq import AutoAWQConfig
from .auto_gptq import AutoGPTQConfig
@@ -161,7 +174,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"inc": INCConfig,
"mxfp4": Mxfp4Config,
"gpt_oss_mxfp4": GptOssMxfp4Config,
"deepseek_v4_fp8": DeepseekV4FP8Config,
"deepseek_v4_fp8": deepseek_config,
"humming": HummingConfig,
"online": OnlineQuantizationConfig,
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
@@ -937,14 +937,13 @@ def w8a8_triton_block_scaled_mm(
assert len(block_size) == 2
block_n, block_k = block_size[0], block_size[1]
# Triton cannot currently bind E8M0 scale tensors directly. On ROCm,
# DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format,
# so decode them to fp32 before launching the kernel.
if current_platform.is_rocm() or current_platform.is_xpu():
if As.dtype == torch.float8_e8m0fnu:
As = _upcast_e8m0_to_fp32(As).contiguous()
if Bs.dtype == torch.float8_e8m0fnu:
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
# Triton cannot currently bind E8M0 scale tensors directly. Checkpoints
# with exponent-only UE8M0 scales (e.g. DeepSeek-V4) are decoded to fp32
# before launching the kernel.
if As.dtype == torch.float8_e8m0fnu:
As = _upcast_e8m0_to_fp32(As).contiguous()
if Bs.dtype == torch.float8_e8m0fnu:
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
assert A.shape[-1] == B.shape[-1]
assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous()
@@ -1089,8 +1088,6 @@ def deepgemm_post_process_weight_scale_block(
is_sfa: bool = False,
) -> torch.Tensor:
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
# DeepGEMM pack the layout expected by the target architecture.
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
@@ -1121,12 +1118,7 @@ def deepgemm_post_process_fp8_weight_block(
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
)
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
if ws.dtype not in (torch.float8_e8m0fnu, torch.uint8):
assert ws.dtype == torch.float32, (
f"Expected tensor scales dtype to be torch.float32 or "
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
@@ -1137,7 +1129,7 @@ def deepgemm_post_process_fp8_weight_block(
if is_bmm:
# Reshape 2D weight/scale to 3D for grouped BMM (einsum):
# wq: (g*r, d) -> (g, r, d)
# ws: (g*r/128, d/128) -> (g, r/128, d/128)
# Scale dimensions follow quant_block_shape.
g = bmm_batch_size
assert wq.ndim == 2 and ws.ndim == 2
d = wq.size(1)
@@ -126,8 +126,14 @@ def _mxfp8_quant_triton_kernel():
amax = tl.maximum(tl.max(tl.abs(x), axis=1), TINY) # [BLOCK_M]
sb = tl.ceil(tl.log2(amax / FP8_MAX)) + 127.0
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
descale = tl.exp2(sb - 127.0)
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
# Scale by the reciprocal rather than dividing by ``exp2(sb - 127)``:
# sb == 0 (an all-zero / denormal block) makes that divisor 2**-127,
# which is subnormal in fp32 and flushes to zero on CDNA, turning the
# block's zeros into 0/0 == NaN. The reciprocal is normal for every
# reachable sb, and both forms are exact powers of two, so nothing
# else changes.
rescale = tl.exp2(127.0 - sb)
xq = (x * rescale[:, None]).to(xq_ptr.dtype.element_ty)
tl.store(
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
xq,
+3 -2
View File
@@ -330,7 +330,7 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
quant_config = getattr(model_config.hf_config, "quantization_config", None)
if quant_config is not None and quant_config.get("quant_method") == "fp8":
model_type = getattr(model_config.hf_config, "model_type", None)
if model_type == "deepseek_v4":
if model_type in ("deepseek_v4", "deepseek_v41"):
model_config.hf_config.quantization_config["quant_method"] = (
"deepseek_v4_fp8"
)
@@ -343,7 +343,7 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
and hf_text_quant_config.get("quant_method") == "fp8"
):
model_type = getattr(model_config.hf_text_config, "model_type", None)
if model_type == "deepseek_v4":
if model_type in ("deepseek_v4", "deepseek_v41"):
model_config.hf_text_config.quantization_config["quant_method"] = (
"deepseek_v4_fp8"
)
@@ -1003,6 +1003,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColQwen3_5": ColQwen3_5Config,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
+8
View File
@@ -376,6 +376,10 @@ _MULTIMODAL_MODELS = {
"vllm.models.deepseek_v4",
"DeepseekV4ForConditionalGeneration",
),
"DeepseekV41ForCausalLM": (
"vllm.models.deepseek_v4_1",
"DeepseekV41ForCausalLM",
),
"Dots3NoteForCausalLM": (
"vllm.models.dots3_note",
"Dots3NoteForCausalLM",
@@ -640,6 +644,10 @@ _SPECULATIVE_DECODING_MODELS = {
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
"DSparkV41DraftModel": (
"vllm.models.deepseek_v4_1",
"DSparkDeepseekV4ForCausalLM",
),
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"K3DSparkModel": (
@@ -173,7 +173,10 @@ def deepseek_v4_mhc_warmup(
# large checkpoints; bail out for any model that is not DeepSeek V4.
config = getattr(model, "config", None)
model_type = getattr(config, "model_type", None) if config is not None else None
if model_type is not None and model_type != "deepseek_v4":
if model_type is not None and model_type not in (
"deepseek_v4",
"deepseek_v41",
):
return
layer = _find_first_mhc_layer(model)
@@ -28,15 +28,20 @@ _DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
"FLASHMLA_SPARSE_DSV4",
"FLASHINFER_MLA_SPARSE_DSV4",
"ROCM_FLASHMLA_SPARSE_DSV4",
"FLASHMLA_SPARSE_DSV41",
"FLASHINFER_MLA_SPARSE_DSV41",
"DEEPSEEK_SPARSE_SWA",
}
)
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_DSV4"})
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset(
{"FLASHINFER_MLA_SPARSE_DSV4", "FLASHINFER_MLA_SPARSE_DSV41"}
)
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
"FLASHINFER_MLA_SPARSE_DSV41": "DSv4.1",
}
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
+2 -2
View File
@@ -58,7 +58,7 @@ from vllm.utils.multi_stream_utils import (
)
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend,
DeepseekV41IndexerBackend,
dsa_indexer_uses_fp4,
get_max_prefill_buffer_size,
)
@@ -1005,7 +1005,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
def forward(self): ...
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV4IndexerBackend
return DeepseekV41IndexerBackend
class DeepseekV4Indexer(nn.Module):
+14 -3
View File
@@ -245,9 +245,20 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
# Block sizes count uncompressed tokens: C4 indexer pages hold 64 rows.
return [256]
class DeepseekV41IndexerBackend(DeepseekV4IndexerBackend):
@staticmethod
def get_name() -> str:
return "DEEPSEEK_V41_INDEXER"
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [64 if current_platform.is_device_capability_family(90) else 128]
@dataclass
class DeepseekV32IndexerPrefillChunkMetadata:
block_table: torch.Tensor
@@ -1239,9 +1250,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
)
block_table = self.indexer_decode_block_table_buffer[:rows, :cols]
seq_lens_is_buffer_view = (use_native and next_n > 1) or (
not use_native and max_decode_len > 1
)
# Flattening always returns a buffer view, including single-token
# batches. Keep its address stable across varlen graph replays.
seq_lens_is_buffer_view = not use_native or next_n > 1
# DCP: localize the now-expanded per-token global bounds to this
# rank's owned KV. Done here (after expansion) so each token's global
+107 -5
View File
@@ -11,7 +11,7 @@ import torch.nn.functional as F
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CUDAGraphMode
from vllm.config import CUDAGraphMode, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -819,6 +819,32 @@ def rocm_fp8_mqa_logits(
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
def _max_decode_logits_rows(num_batched_tokens: int) -> int:
"""Upper bound on decode rows the paged-MQA logits buffer can ever hold.
``rocm_fp8_paged_mqa_logits`` sizes its workspace as
``(batch_size * next_n, max_model_len)``. ``batch_size`` is bounded by
``max_num_seqs`` and ``next_n`` by ``1 + num_speculative_tokens``, which is
far tighter than ``max_num_batched_tokens`` -- 192 vs 16384 for a typical
32-seq DSpark-5 deployment. The loose bound is harmless at short contexts
but scales with ``max_model_len``, so at the model's full context it asks
for tens of TiB and the engine cannot start. Take whichever valid bound is
smaller; the workspace is locked after profiling, so it must not be under-
estimated.
"""
try:
vllm_config = get_current_vllm_config()
except Exception:
return num_batched_tokens
scheduler_config = getattr(vllm_config, "scheduler_config", None)
max_num_seqs = getattr(scheduler_config, "max_num_seqs", None)
if not max_num_seqs:
return num_batched_tokens
speculative_config = getattr(vllm_config, "speculative_config", None)
num_spec = getattr(speculative_config, "num_speculative_tokens", 0) or 0
return min(num_batched_tokens, max_num_seqs * (1 + num_spec))
def rocm_aiter_sparse_attn_indexer_fake(
hidden_states: torch.Tensor,
k_cache_prefix: LayerNameType,
@@ -835,6 +861,9 @@ def rocm_aiter_sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool = False,
compress_ratio: int = 1,
candidate_blocks: torch.Tensor | None = None,
candidate_block_size: int = 0,
candidate_write: bool = False,
) -> torch.Tensor:
return topk_indices_buffer
@@ -856,6 +885,9 @@ def rocm_aiter_sparse_attn_indexer(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool = False,
compress_ratio: int = 1,
candidate_blocks: torch.Tensor | None = None,
candidate_block_size: int = 0,
candidate_write: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
forward_context = get_forward_context()
@@ -881,15 +913,15 @@ def rocm_aiter_sparse_attn_indexer(
)
# Decode logits buffer, used by rocm_fp8_paged_mqa_logits.
# batch_size * next_n <= hidden_states.shape[0] == max_num_batched_tokens
decode_rows = _max_decode_logits_rows(hidden_states.shape[0])
if _ON_GFX942 or _ON_GFX950:
workspace_manager.get_simultaneous(
((hidden_states.shape[0], max_model_len), torch.float32),
((decode_rows, max_model_len), torch.float32),
)
else:
workspace_manager.get_simultaneous(
(
(q_fp8.shape[1], hidden_states.shape[0], max_model_len),
(q_fp8.shape[1], decode_rows, max_model_len),
torch.float32,
),
)
@@ -919,6 +951,9 @@ def rocm_aiter_sparse_attn_indexer(
topk_indices_buffer,
skip_k_cache_insert,
compress_ratio,
candidate_blocks,
candidate_block_size,
candidate_write,
)
layer_attn_metadata = attn_metadata[k_cache_prefix]
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
@@ -974,6 +1009,30 @@ def rocm_aiter_sparse_attn_indexer(
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
if candidate_blocks is not None:
from vllm.model_executor.layers.sparse_attn_indexer import (
_apply_candidate_mask,
_select_candidate_blocks,
)
chunk_candidates = candidate_blocks[chunk.token_start : chunk.token_end]
if candidate_write:
_select_candidate_blocks(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
chunk_candidates.shape[1],
candidate_block_size,
chunk_candidates,
)
else:
_apply_candidate_mask(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
chunk_candidates,
candidate_block_size,
)
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
@@ -1041,6 +1100,37 @@ def rocm_aiter_sparse_attn_indexer(
max_model_len=max_model_len,
)
if candidate_blocks is not None:
from vllm.model_executor.layers.sparse_attn_indexer import (
_apply_candidate_mask,
_select_candidate_blocks,
)
num_rows = logits.shape[0]
visible = decode_metadata.seq_lens.reshape(-1)
if visible.numel() != num_rows:
visible = visible.repeat_interleave(next_n)
visible = visible[:num_rows].to(torch.int64)
row_starts = torch.zeros_like(visible)
decode_candidates = candidate_blocks[:num_rows]
if candidate_write:
_select_candidate_blocks(
logits,
row_starts,
visible,
decode_candidates.shape[1],
candidate_block_size,
decode_candidates,
)
else:
_apply_candidate_mask(
logits,
row_starts,
visible,
decode_candidates,
candidate_block_size,
)
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
num_rows = logits.shape[0]
@@ -1097,6 +1187,10 @@ def _decode_e8m0_scales(scale: torch.Tensor) -> torch.Tensor:
)
return _upcast_e8m0_to_fp32(scale).contiguous()
if scale.dtype == torch.uint8:
# MXFP8 parameters preserve E8M0 scales as their raw exponent bytes.
# They are biased exponents, not numeric uint8 scale values.
return torch.exp2(scale.to(torch.int16).to(torch.float32) - 127.0)
return scale.to(torch.float32)
@@ -1226,7 +1320,15 @@ def _get_cached_wo_a_bf16(
)
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
if wo_a_scale_param is not None:
if wo_a_scale_param is None:
# ModelOpt MXFP8 stores the multiplicative E8M0 scale without the
# historical ``_inv`` suffix.
wo_a_scale_param = getattr(wo_a, "weight_scale", None)
# Emulated MXFP8 kernels can replace the original one-byte weight with an
# already-dequantized BF16 tensor while retaining the scale attribute for
# metadata. Applying that retained scale again would double-dequantize the
# weight. Block scaling is only valid while the one-byte FP8 storage remains.
if wo_a_scale_param is not None and wo_a.weight.element_size() == 1:
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
torch.float32
)
+23 -14
View File
@@ -22,6 +22,7 @@ from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.kv_cache_interface import (
AttentionSpec,
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KpoolTailSpec,
@@ -2013,15 +2014,22 @@ def _get_packed_kv_cache_groups(
cdiv(len(names), n) * page for page, names in page_size_layers.items()
)
# Bytes a block must hold however the mamba buckets end up split: a mamba
# bucket can go down to one state per group, every other bucket's split is
# already fixed by the repeat pattern.
# Bytes a block must hold however the state buckets end up split: mamba,
# circular-buffer and unsplit sliding-window buckets can go down to one
# state per group, every other bucket's split is fixed by the repeat pattern.
def is_state_bucket(spec: UniformTypeKVCacheSpecs) -> bool:
if isinstance(spec.first_spec, (MambaSpec, CircularBufferSpec)):
return True
return repeats_per_group is None and isinstance(
spec.first_spec, SlidingWindowSpec
)
anchor_bytes = max(
(
widest_group_bytes(
page_size_layers,
len(spec.kv_cache_specs)
if isinstance(spec.first_spec, MambaSpec)
if is_state_bucket(spec)
else num_groups_for(spec, balanced),
)
for spec, page_size_layers, balanced in bucketed
@@ -2032,10 +2040,9 @@ def _get_packed_kv_cache_groups(
groups = []
for spec, page_size_layers, balanced in bucketed:
num_groups = num_groups_for(spec, balanced)
# `_align_hybrid_block_size` pads a mamba state up to one attention
# page, so cap a mamba group at the states a block already fits rather
# than let it widen the block.
if anchor_bytes and isinstance(spec.first_spec, MambaSpec):
# Cap a state group at the states a block already fits rather than let
# it widen the block.
if anchor_bytes and is_state_bucket(spec):
states_per_block = max(anchor_bytes // spec.first_spec.page_size_bytes, 1)
num_groups = max(
num_groups, cdiv(len(spec.kv_cache_specs), states_per_block)
@@ -2071,8 +2078,9 @@ def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool:
if spec_config is None or not spec_config.use_eagle():
return False
model_config = vllm_config.model_config
return (
model_config is not None and model_config.hf_config.model_type == "deepseek_v4"
return model_config is not None and model_config.hf_config.model_type in (
"deepseek_v4",
"deepseek_v41",
)
@@ -2093,9 +2101,9 @@ def _annotate_eagle_groups(
spec merging, wherever grouping happens to land. It is sufficient but
not necessary: a drafter whose spec is indistinguishable from the
target's cannot be found this way.
2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses
the target's own decoder layer and so carries no spec marker. Its draft
attention layer is always the last registered layer, so flag whichever
2. Model-scoped positional fallback for DeepseekV4/V4.1, whose MTP block
reuses the target's own decoder layer and so carries no spec marker. Its
draft attention layer is always the last registered layer, so flag whichever
group holds it. This rule is only valid where the groups partition
exactly the layers of ``kv_cache_spec``, which is true on the packed
grouping path and not in general; other callers must leave
@@ -2108,7 +2116,8 @@ def _annotate_eagle_groups(
kv_cache_spec: The kv cache spec of each attention layer, in layer
registration order. Only read by rule 2.
kv_cache_groups: Groups to annotate in place.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed
group.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle_block_drop():
+26 -8
View File
@@ -51,7 +51,7 @@ from vllm.v1.simple_kv_offload.metadata import (
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
from vllm.v1.core.kv_cache_utils import BlockHashList, KVCacheBlock
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
@@ -783,6 +783,18 @@ class SimpleCPUOffloadScheduler:
req_ids = list(dict.fromkeys(req_ids))
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta
def _cached_gpu_block(
self, resolved_hashes: "BlockHashList", block_idx: int, group_id: int
) -> "KVCacheBlock | None":
"""Return the GPU block still cached under this group's block hash."""
if block_idx >= len(resolved_hashes):
return None
assert self._gpu_block_pool is not None
blocks = self._gpu_block_pool.get_cached_block(
resolved_hashes[block_idx], [group_id]
)
return blocks[0] if blocks else None
def _select_eager_blocks_to_store(
self,
state: StoreRequestState,
@@ -827,18 +839,25 @@ class SimpleCPUOffloadScheduler:
# the middle of the request.
group_size = self.group_block_sizes[g]
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
resolved_hashes = (
resolve_block_hashes(
request.block_hashes, self.hash_block_size, group_size
)
if block_meta is not None
else None
resolved_hashes = resolve_block_hashes(
request.block_hashes, self.hash_block_size, group_size
)
curr_mm_idx = 0
secondary_mm_idx = 0
start = state.num_stored_blocks[g]
for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start):
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
if gpu_block.is_null:
# Sliding-window groups null pages that left the window
# before the connector sees the block table, but the
# retained prefix-cache tail stays hashed in the GPU free
# queue; store it from there.
recovered = self._cached_gpu_block(resolved_hashes, i, g)
if recovered is None:
advanced_per_group[g] += 1
continue
gpu_block = recovered
gpu_block_id = gpu_block.block_id
if (
self._classify_store_candidate(gpu_block)
is not _StoreAdmission.READY
@@ -854,7 +873,6 @@ class SimpleCPUOffloadScheduler:
if block_meta is not None:
token_start = i * group_size
token_end = token_start + group_size
assert resolved_hashes is not None
parent_hash = (
None
if i == 0
@@ -17,9 +17,9 @@ def _resolve_dspark_attention_backend(
) -> AttentionBackendEnum | None:
if draft_backend is not None:
return draft_backend
# DeepSeek-V4 draft layers share the target's KV-cache layout. Other
# DeepSeek-V4(.1) draft layers share the target's KV-cache layout. Other
# DSpark architectures may use a different attention kind.
if draft_model_config.hf_config.model_type == "deepseek_v4":
if draft_model_config.hf_config.model_type in ("deepseek_v4", "deepseek_v41"):
if target_backend is not None:
logger.info_once(
"Using the target model's %s attention backend for the "
@@ -112,7 +112,14 @@ def get_eagle3_aux_layers_from_config(
if not layer_ids:
dspark_layer_ids = getattr(hf_config, "dspark_target_layer_ids", None)
if dspark_layer_ids:
layer_ids = [i + 1 for i in dspark_layer_ids]
if getattr(hf_config, "model_type", None) == "deepseek_v41":
# v4.1 reads the attention *inputs* of its target layers, and
# the target model captures the entry stream of layer L when
# idx+1 == L, so the ids are used as-is. (v4's ids are in
# capture-after semantics and keep the +1.)
layer_ids = list(dspark_layer_ids)
else:
layer_ids = [i + 1 for i in dspark_layer_ids]
if not layer_ids:
# Dense DSpark (e.g. Qwen3) also uses different aux layer semantics.
target_layer_ids = getattr(hf_config, "target_layer_ids", None)
+38 -3
View File
@@ -813,6 +813,9 @@ class GPUModelRunner(
# Persistent buffers for CUDA graphs.
self.input_ids = self._make_buffer(self.max_num_tokens, dtype=torch.int32)
# [max_num_reqs, depth] prompt token ids preceding each request's chunk
# start, for models exposing `token_lookback_depth`. Sized after load.
self.lookback_token_ids: CpuGpuBuffer | None = None
self.positions = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=self.device
)
@@ -1065,9 +1068,36 @@ class GPUModelRunner(
)
return self._mamba_bufs
def _init_model_kwargs(self):
def _prepare_lookback_token_ids(self, num_reqs: int) -> torch.Tensor:
"""Gather, per request, the `depth` prompt token ids preceding its
first scheduled token (column j is position start - 1 - j); -1 where
the position is before the prompt or already past it. Generated
positions are left to the model: under async scheduling the CPU token
table holds placeholders for them."""
buf = self.lookback_token_ids
assert buf is not None
buf.np.fill(-1)
if num_reqs > 0:
depth = buf.np.shape[1]
starts = self.input_batch.num_computed_tokens_cpu[:num_reqs, None]
pos = starts - np.arange(1, depth + 1)
num_prompt = self.input_batch.num_prompt_tokens[:num_reqs, None]
valid = (pos >= 0) & (pos < num_prompt)
rows = np.arange(num_reqs)[:, None]
ids = self.input_batch.token_ids_cpu[rows, np.clip(pos, 0, None)]
buf.np[:num_reqs] = np.where(valid, ids, -1)
return buf.copy_to_gpu()
def _init_model_kwargs(self, num_reqs: int | None = None):
model_kwargs = dict[str, Any]()
if self.lookback_token_ids is not None:
if num_reqs is None:
num_reqs = self.input_batch.num_reqs
model_kwargs["lookback_token_ids"] = self._prepare_lookback_token_ids(
num_reqs
)
if not self.is_pooling_model:
return model_kwargs
@@ -5314,6 +5344,11 @@ class GPUModelRunner(
self.model = model_loader.load_model(
vllm_config=self.vllm_config, model_config=self.model_config
)
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
if lookback_depth > 0:
self.lookback_token_ids = self._make_buffer(
self.max_num_reqs, lookback_depth, dtype=torch.int32
)
if self.lora_config:
self.model = self.load_lora_model(
self.model, self.vllm_config, self.device
@@ -6076,7 +6111,7 @@ class GPUModelRunner(
):
# Make sure padding doesn't exceed max_num_tokens
assert num_tokens_padded <= self.max_num_tokens
model_kwargs = self._init_model_kwargs()
model_kwargs = self._init_model_kwargs(num_reqs=0)
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
input_ids, inputs_embeds = self._prepare_mm_inputs(num_tokens_padded)
@@ -6087,7 +6122,7 @@ class GPUModelRunner(
elif self.enable_prompt_embeds:
input_ids = None
inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded]
model_kwargs = self._init_model_kwargs()
model_kwargs = self._init_model_kwargs(num_reqs=0)
else:
input_ids = self.input_ids.gpu[:num_tokens_padded]
inputs_embeds = None