[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
+6
-4
@@ -2143,13 +2143,15 @@ class rocm_aiter_ops:
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
from vllm.platforms.rocm import on_gfx11, on_gfx950
|
||||
|
||||
if on_gfx11() and not _OPS_REGISTERED:
|
||||
# This op has self-contained Triton/C++ implementations on gfx11
|
||||
# and gfx950. Only its optional top-k fast path comes from aiter.
|
||||
if (on_gfx11() or on_gfx950()) and not _OPS_REGISTERED:
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_sparse_attn_indexer",
|
||||
op_func=rocm_aiter_sparse_attn_indexer,
|
||||
mutates_args=["topk_indices_buffer"],
|
||||
mutates_args=["topk_indices_buffer", "candidate_blocks"],
|
||||
fake_impl=rocm_aiter_sparse_attn_indexer_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
@@ -2327,7 +2329,7 @@ class rocm_aiter_ops:
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_sparse_attn_indexer",
|
||||
op_func=rocm_aiter_sparse_attn_indexer,
|
||||
mutates_args=["topk_indices_buffer"],
|
||||
mutates_args=["topk_indices_buffer", "candidate_blocks"],
|
||||
fake_impl=rocm_aiter_sparse_attn_indexer_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.forward_context import (
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import weak_ref_tensor, weak_ref_tensors
|
||||
@@ -56,6 +57,12 @@ def is_breakable_cudagraph_enabled() -> bool:
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def _weak_ref_capture_arg(arg: Any) -> Any:
|
||||
if isinstance(arg, QuantizedActivation):
|
||||
return arg.weak_ref()
|
||||
return weak_ref_tensor(arg)
|
||||
|
||||
|
||||
def eager_break_during_capture(fn: F) -> F:
|
||||
"""Decorator that turns a custom-op Python kernel into a "break point"
|
||||
for the breakable cudagraph capture.
|
||||
@@ -105,13 +112,8 @@ def eager_break_during_capture(fn: F) -> F:
|
||||
# Weak-ref args: strong refs in the replay lambda pin cudagraph-pool
|
||||
# slots across batch descriptors. cudagraph owns the slot, so the
|
||||
# weak_ref is safe to deref on replay.
|
||||
weak_args = tuple(
|
||||
weak_ref_tensor(a) if isinstance(a, torch.Tensor) else a for a in args
|
||||
)
|
||||
weak_kwargs = {
|
||||
k: weak_ref_tensor(v) if isinstance(v, torch.Tensor) else v
|
||||
for k, v in kwargs.items()
|
||||
}
|
||||
weak_args = tuple(_weak_ref_capture_arg(a) for a in args)
|
||||
weak_kwargs = {k: _weak_ref_capture_arg(v) for k, v in kwargs.items()}
|
||||
return capture.add_eager(lambda: fn(*weak_args, **weak_kwargs))
|
||||
|
||||
return wrapper # type: ignore[return-value]
|
||||
|
||||
+19
-9
@@ -15,6 +15,14 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Architecture -> the hf_text_config field naming its n-gram layers. A model is
|
||||
# only configurable here if it actually has such layers to store.
|
||||
_NGRAM_LAYER_FIELDS = {
|
||||
"DeepseekV41ForCausalLM": "engram_layer_ids",
|
||||
"Qwen4ExpForCausalLM": "ple_layer_ids",
|
||||
"Qwen4ExpForConditionalGeneration": "ple_layer_ids",
|
||||
}
|
||||
|
||||
|
||||
def _default_cpu_offload() -> bool:
|
||||
"""Honor the legacy environment variable only when the field is omitted."""
|
||||
@@ -40,23 +48,25 @@ class EngramConfig:
|
||||
Otherwise, each DP rank has a separate TP-sharded embedding replica."""
|
||||
|
||||
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
|
||||
"""Reject Engram configuration for models without supported embeddings."""
|
||||
"""Reject Engram configuration for models without n-gram embeddings."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
supported_architectures = {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}
|
||||
field = (
|
||||
_NGRAM_LAYER_FIELDS.get(model_config.architecture)
|
||||
if model_config is not None
|
||||
else None
|
||||
)
|
||||
if (
|
||||
model_config is None
|
||||
or model_config.architecture not in supported_architectures
|
||||
or field is None
|
||||
or not current_platform.is_cuda()
|
||||
or not getattr(model_config.hf_text_config, "ple_layer_ids", None)
|
||||
or not getattr(model_config.hf_text_config, field, None)
|
||||
):
|
||||
raise ValueError(
|
||||
"EngramConfig requires a model with supported Engram "
|
||||
"embeddings. Currently only the CUDA Qwen4Exp implementation "
|
||||
"with non-empty ple_layer_ids is supported."
|
||||
"embeddings. Currently only the CUDA Qwen4Exp and DeepSeek "
|
||||
"V4.1 implementations with non-empty n-gram layer ids are "
|
||||
"supported."
|
||||
)
|
||||
|
||||
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
|
||||
|
||||
@@ -165,6 +165,7 @@ FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTPModel",
|
||||
"DeepseekV41ForCausalLM",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -679,12 +679,18 @@ class SpeculativeConfig:
|
||||
],
|
||||
}
|
||||
)
|
||||
if hf_config.model_type == "deepseek_v4":
|
||||
if hf_config.model_type in ("deepseek_v4", "deepseek_v41"):
|
||||
# V4.1 has no classic-MTP draft: its checkpoints ship DSpark stages
|
||||
# under ``mtp.*``, so only V4 gets an MTP architecture here. The
|
||||
# DSpark path rewrites ``architectures`` itself and needs only
|
||||
# ``n_predict``; ``method="mtp"`` on V4.1 is rejected below.
|
||||
is_v41 = hf_config.model_type == "deepseek_v41"
|
||||
hf_config.model_type = "deepseek_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["DeepSeekV4MTPModel"]}
|
||||
)
|
||||
overrides = {"n_predict": n_predict}
|
||||
if not is_v41:
|
||||
overrides["architectures"] = ["DeepSeekV4MTPModel"]
|
||||
hf_config.update(overrides)
|
||||
if hf_config.model_type in ("pangu_ultra_moe"):
|
||||
hf_config.model_type = "pangu_ultra_moe_mtp"
|
||||
if hf_config.model_type == "pangu_ultra_moe_mtp":
|
||||
@@ -1335,6 +1341,18 @@ class SpeculativeConfig:
|
||||
MTPModelTypes
|
||||
):
|
||||
self.method = "mtp"
|
||||
if (
|
||||
self.target_model_config is not None
|
||||
and self.target_model_config.hf_config.model_type
|
||||
== "deepseek_v41"
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek V4.1 has no classic-MTP draft: its "
|
||||
"checkpoints ship DSpark stages under mtp.* "
|
||||
"(main_proj/markov_head/confidence_head) and carry "
|
||||
"no e_proj/h_proj/enorm/hnorm/hc_head weights. Use "
|
||||
"speculative method 'dspark' instead of 'mtp'."
|
||||
)
|
||||
if (
|
||||
self.num_speculative_tokens > 1
|
||||
and self.draft_model_config.hf_config.model_type
|
||||
@@ -1398,12 +1416,26 @@ class SpeculativeConfig:
|
||||
and "Gemma4DSparkModel" not in self.draft_model_config.architectures
|
||||
and "K3DSparkModel" not in self.draft_model_config.architectures
|
||||
):
|
||||
# DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config
|
||||
# DeepSeek-V4(.1) DSpark reuses the full target config
|
||||
# and its weights ship in the target checkpoint.
|
||||
self.draft_model_config.hf_config.model_type = "deepseek_v4"
|
||||
self.draft_model_config.hf_config.architectures = [
|
||||
"DSparkDraftModel"
|
||||
is_v41 = (
|
||||
self.target_model_config.hf_config.model_type == "deepseek_v41"
|
||||
)
|
||||
draft_hf_config = self.draft_model_config.hf_config
|
||||
draft_hf_config.model_type = (
|
||||
"deepseek_v41" if is_v41 else "deepseek_v4"
|
||||
)
|
||||
draft_hf_config.architectures = [
|
||||
"DSparkV41DraftModel" if is_v41 else "DSparkDraftModel"
|
||||
]
|
||||
if is_v41:
|
||||
# hf_config_override set n_predict to the number of
|
||||
# MTP stages (3), but one DSpark round drafts
|
||||
# dspark_block_size tokens; num_speculative_tokens
|
||||
# divisibility is checked against n_predict below.
|
||||
draft_hf_config.n_predict = getattr(
|
||||
draft_hf_config, "dspark_block_size", None
|
||||
) or getattr(draft_hf_config, "n_predict", None)
|
||||
self.draft_model_config.quantization = (
|
||||
self.target_model_config.quantization
|
||||
)
|
||||
|
||||
@@ -81,6 +81,7 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
"DeepSeekV4MTPModel",
|
||||
"DeepseekV41ForCausalLM",
|
||||
"Dots3NoteForCausalLM",
|
||||
"Dots3NoteMTPModel",
|
||||
"Glm5NextForCausalLM",
|
||||
|
||||
@@ -91,6 +91,7 @@ class MooncakeStoreCoordinator:
|
||||
assert all(
|
||||
scheduler_block_size % g.kv_cache_spec.block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.mamba_group_ids = {
|
||||
|
||||
@@ -460,11 +460,13 @@ class NixlBaseConnectorWorker:
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
]
|
||||
if len(ple_groups) != 1 or len(ple_groups[0][1].layer_names) != 1:
|
||||
if len(ple_groups) > 1 or any(
|
||||
len(group.layer_names) != 1 for _, group in ple_groups
|
||||
):
|
||||
raise ValueError(
|
||||
"CSA-linear NIXL requires exactly one PLE cache owner."
|
||||
"CSA-linear NIXL requires at most one PLE cache owner."
|
||||
)
|
||||
self._ple_group_index = ple_groups[0][0]
|
||||
self._ple_group_index = ple_groups[0][0] if ple_groups else None
|
||||
|
||||
if self._has_mamba:
|
||||
assert self._is_hma_required
|
||||
|
||||
@@ -8,7 +8,7 @@ expose_input_quant_key; the kernel validates and reads the activation via
|
||||
as_quantized_activation.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -34,6 +34,16 @@ class QuantizedActivation:
|
||||
orig_shape: torch.Size
|
||||
quant_key: QuantKey
|
||||
|
||||
def weak_ref(self) -> "QuantizedActivation":
|
||||
"""Return a copy with non-owning tensor references for CUDA graph replay."""
|
||||
from vllm.utils.torch_utils import weak_ref_tensor
|
||||
|
||||
return replace(
|
||||
self,
|
||||
data=weak_ref_tensor(self.data),
|
||||
scale=weak_ref_tensor(self.scale),
|
||||
)
|
||||
|
||||
|
||||
def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
|
||||
"""Advertise the kernel's pre-quantized input key on the layer, if any.
|
||||
|
||||
@@ -115,7 +115,20 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
# lazy import to avoid triggering `torch.compile` too early
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
if current_platform.is_cuda() or current_platform.is_rocm():
|
||||
# The v4.1 class is the v4 one extended to accept model_type
|
||||
# "deepseek_v41" and its 32x32 MXFP8 linear layout. V4.1 has
|
||||
# platform-specific implementations for both CUDA and ROCm.
|
||||
from vllm.models.deepseek_v4_1 import (
|
||||
DeepseekV4FP8Config as DeepseekV41FP8Config,
|
||||
)
|
||||
|
||||
deepseek_config: type[QuantizationConfig] = DeepseekV41FP8Config
|
||||
else:
|
||||
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
deepseek_config = DeepseekV4FP8Config
|
||||
|
||||
from .auto_awq import AutoAWQConfig
|
||||
from .auto_gptq import AutoGPTQConfig
|
||||
@@ -161,7 +174,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"inc": INCConfig,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"gpt_oss_mxfp4": GptOssMxfp4Config,
|
||||
"deepseek_v4_fp8": DeepseekV4FP8Config,
|
||||
"deepseek_v4_fp8": deepseek_config,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
|
||||
|
||||
@@ -937,14 +937,13 @@ def w8a8_triton_block_scaled_mm(
|
||||
assert len(block_size) == 2
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
|
||||
# Triton cannot currently bind E8M0 scale tensors directly. On ROCm,
|
||||
# DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format,
|
||||
# so decode them to fp32 before launching the kernel.
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
|
||||
# Triton cannot currently bind E8M0 scale tensors directly. Checkpoints
|
||||
# with exponent-only UE8M0 scales (e.g. DeepSeek-V4) are decoded to fp32
|
||||
# before launching the kernel.
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
|
||||
|
||||
assert A.shape[-1] == B.shape[-1]
|
||||
assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous()
|
||||
@@ -1089,8 +1088,6 @@ def deepgemm_post_process_weight_scale_block(
|
||||
is_sfa: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
|
||||
# DeepGEMM pack the layout expected by the target architecture.
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
@@ -1121,12 +1118,7 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
|
||||
)
|
||||
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
|
||||
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
|
||||
# (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
if ws.dtype not in (torch.float8_e8m0fnu, torch.uint8):
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
|
||||
@@ -1137,7 +1129,7 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
if is_bmm:
|
||||
# Reshape 2D weight/scale to 3D for grouped BMM (einsum):
|
||||
# wq: (g*r, d) -> (g, r, d)
|
||||
# ws: (g*r/128, d/128) -> (g, r/128, d/128)
|
||||
# Scale dimensions follow quant_block_shape.
|
||||
g = bmm_batch_size
|
||||
assert wq.ndim == 2 and ws.ndim == 2
|
||||
d = wq.size(1)
|
||||
|
||||
@@ -126,8 +126,14 @@ def _mxfp8_quant_triton_kernel():
|
||||
amax = tl.maximum(tl.max(tl.abs(x), axis=1), TINY) # [BLOCK_M]
|
||||
sb = tl.ceil(tl.log2(amax / FP8_MAX)) + 127.0
|
||||
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
|
||||
descale = tl.exp2(sb - 127.0)
|
||||
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
|
||||
# Scale by the reciprocal rather than dividing by ``exp2(sb - 127)``:
|
||||
# sb == 0 (an all-zero / denormal block) makes that divisor 2**-127,
|
||||
# which is subnormal in fp32 and flushes to zero on CDNA, turning the
|
||||
# block's zeros into 0/0 == NaN. The reciprocal is normal for every
|
||||
# reachable sb, and both forms are exact powers of two, so nothing
|
||||
# else changes.
|
||||
rescale = tl.exp2(127.0 - sb)
|
||||
xq = (x * rescale[:, None]).to(xq_ptr.dtype.element_ty)
|
||||
tl.store(
|
||||
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
|
||||
xq,
|
||||
|
||||
@@ -330,7 +330,7 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
if quant_config is not None and quant_config.get("quant_method") == "fp8":
|
||||
model_type = getattr(model_config.hf_config, "model_type", None)
|
||||
if model_type == "deepseek_v4":
|
||||
if model_type in ("deepseek_v4", "deepseek_v41"):
|
||||
model_config.hf_config.quantization_config["quant_method"] = (
|
||||
"deepseek_v4_fp8"
|
||||
)
|
||||
@@ -343,7 +343,7 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
and hf_text_quant_config.get("quant_method") == "fp8"
|
||||
):
|
||||
model_type = getattr(model_config.hf_text_config, "model_type", None)
|
||||
if model_type == "deepseek_v4":
|
||||
if model_type in ("deepseek_v4", "deepseek_v41"):
|
||||
model_config.hf_text_config.quantization_config["quant_method"] = (
|
||||
"deepseek_v4_fp8"
|
||||
)
|
||||
@@ -1003,6 +1003,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
|
||||
@@ -376,6 +376,10 @@ _MULTIMODAL_MODELS = {
|
||||
"vllm.models.deepseek_v4",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
),
|
||||
"DeepseekV41ForCausalLM": (
|
||||
"vllm.models.deepseek_v4_1",
|
||||
"DeepseekV41ForCausalLM",
|
||||
),
|
||||
"Dots3NoteForCausalLM": (
|
||||
"vllm.models.dots3_note",
|
||||
"Dots3NoteForCausalLM",
|
||||
@@ -640,6 +644,10 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"DSparkV41DraftModel": (
|
||||
"vllm.models.deepseek_v4_1",
|
||||
"DSparkDeepseekV4ForCausalLM",
|
||||
),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
|
||||
@@ -173,7 +173,10 @@ def deepseek_v4_mhc_warmup(
|
||||
# large checkpoints; bail out for any model that is not DeepSeek V4.
|
||||
config = getattr(model, "config", None)
|
||||
model_type = getattr(config, "model_type", None) if config is not None else None
|
||||
if model_type is not None and model_type != "deepseek_v4":
|
||||
if model_type is not None and model_type not in (
|
||||
"deepseek_v4",
|
||||
"deepseek_v41",
|
||||
):
|
||||
return
|
||||
|
||||
layer = _find_first_mhc_layer(model)
|
||||
|
||||
@@ -28,15 +28,20 @@ _DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
|
||||
"FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHMLA_SPARSE_DSV41",
|
||||
"FLASHINFER_MLA_SPARSE_DSV41",
|
||||
"DEEPSEEK_SPARSE_SWA",
|
||||
}
|
||||
)
|
||||
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
|
||||
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_DSV4"})
|
||||
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset(
|
||||
{"FLASHINFER_MLA_SPARSE_DSV4", "FLASHINFER_MLA_SPARSE_DSV41"}
|
||||
)
|
||||
|
||||
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
|
||||
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV41": "DSv4.1",
|
||||
}
|
||||
|
||||
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
|
||||
|
||||
@@ -58,7 +58,7 @@ from vllm.utils.multi_stream_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepseekV41IndexerBackend,
|
||||
dsa_indexer_uses_fp4,
|
||||
get_max_prefill_buffer_size,
|
||||
)
|
||||
@@ -1005,7 +1005,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
def forward(self): ...
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV4IndexerBackend
|
||||
return DeepseekV41IndexerBackend
|
||||
|
||||
|
||||
class DeepseekV4Indexer(nn.Module):
|
||||
|
||||
@@ -245,9 +245,20 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
# Block sizes count uncompressed tokens: C4 indexer pages hold 64 rows.
|
||||
return [256]
|
||||
|
||||
|
||||
class DeepseekV41IndexerBackend(DeepseekV4IndexerBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "DEEPSEEK_V41_INDEXER"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [64 if current_platform.is_device_capability_family(90) else 128]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV32IndexerPrefillChunkMetadata:
|
||||
block_table: torch.Tensor
|
||||
@@ -1239,9 +1250,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
block_table = self.indexer_decode_block_table_buffer[:rows, :cols]
|
||||
|
||||
seq_lens_is_buffer_view = (use_native and next_n > 1) or (
|
||||
not use_native and max_decode_len > 1
|
||||
)
|
||||
# Flattening always returns a buffer view, including single-token
|
||||
# batches. Keep its address stable across varlen graph replays.
|
||||
seq_lens_is_buffer_view = not use_native or next_n > 1
|
||||
|
||||
# DCP: localize the now-expanded per-token global bounds to this
|
||||
# rank's owned KV. Done here (after expansion) so each token's global
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch.nn.functional as F
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
@@ -819,6 +819,32 @@ def rocm_fp8_mqa_logits(
|
||||
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||||
|
||||
|
||||
def _max_decode_logits_rows(num_batched_tokens: int) -> int:
|
||||
"""Upper bound on decode rows the paged-MQA logits buffer can ever hold.
|
||||
|
||||
``rocm_fp8_paged_mqa_logits`` sizes its workspace as
|
||||
``(batch_size * next_n, max_model_len)``. ``batch_size`` is bounded by
|
||||
``max_num_seqs`` and ``next_n`` by ``1 + num_speculative_tokens``, which is
|
||||
far tighter than ``max_num_batched_tokens`` -- 192 vs 16384 for a typical
|
||||
32-seq DSpark-5 deployment. The loose bound is harmless at short contexts
|
||||
but scales with ``max_model_len``, so at the model's full context it asks
|
||||
for tens of TiB and the engine cannot start. Take whichever valid bound is
|
||||
smaller; the workspace is locked after profiling, so it must not be under-
|
||||
estimated.
|
||||
"""
|
||||
try:
|
||||
vllm_config = get_current_vllm_config()
|
||||
except Exception:
|
||||
return num_batched_tokens
|
||||
scheduler_config = getattr(vllm_config, "scheduler_config", None)
|
||||
max_num_seqs = getattr(scheduler_config, "max_num_seqs", None)
|
||||
if not max_num_seqs:
|
||||
return num_batched_tokens
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
num_spec = getattr(speculative_config, "num_speculative_tokens", 0) or 0
|
||||
return min(num_batched_tokens, max_num_seqs * (1 + num_spec))
|
||||
|
||||
|
||||
def rocm_aiter_sparse_attn_indexer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
@@ -835,6 +861,9 @@ def rocm_aiter_sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
@@ -856,6 +885,9 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
forward_context = get_forward_context()
|
||||
@@ -881,15 +913,15 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
)
|
||||
|
||||
# Decode logits buffer, used by rocm_fp8_paged_mqa_logits.
|
||||
# batch_size * next_n <= hidden_states.shape[0] == max_num_batched_tokens
|
||||
decode_rows = _max_decode_logits_rows(hidden_states.shape[0])
|
||||
if _ON_GFX942 or _ON_GFX950:
|
||||
workspace_manager.get_simultaneous(
|
||||
((hidden_states.shape[0], max_model_len), torch.float32),
|
||||
((decode_rows, max_model_len), torch.float32),
|
||||
)
|
||||
else:
|
||||
workspace_manager.get_simultaneous(
|
||||
(
|
||||
(q_fp8.shape[1], hidden_states.shape[0], max_model_len),
|
||||
(q_fp8.shape[1], decode_rows, max_model_len),
|
||||
torch.float32,
|
||||
),
|
||||
)
|
||||
@@ -919,6 +951,9 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
compress_ratio,
|
||||
candidate_blocks,
|
||||
candidate_block_size,
|
||||
candidate_write,
|
||||
)
|
||||
layer_attn_metadata = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
|
||||
@@ -974,6 +1009,30 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
if candidate_blocks is not None:
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_apply_candidate_mask,
|
||||
_select_candidate_blocks,
|
||||
)
|
||||
|
||||
chunk_candidates = candidate_blocks[chunk.token_start : chunk.token_end]
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
chunk_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
chunk_candidates,
|
||||
)
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
chunk_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
@@ -1041,6 +1100,37 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
|
||||
if candidate_blocks is not None:
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_apply_candidate_mask,
|
||||
_select_candidate_blocks,
|
||||
)
|
||||
|
||||
num_rows = logits.shape[0]
|
||||
visible = decode_metadata.seq_lens.reshape(-1)
|
||||
if visible.numel() != num_rows:
|
||||
visible = visible.repeat_interleave(next_n)
|
||||
visible = visible[:num_rows].to(torch.int64)
|
||||
row_starts = torch.zeros_like(visible)
|
||||
decode_candidates = candidate_blocks[:num_rows]
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
row_starts,
|
||||
visible,
|
||||
decode_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
decode_candidates,
|
||||
)
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
row_starts,
|
||||
visible,
|
||||
decode_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
@@ -1097,6 +1187,10 @@ def _decode_e8m0_scales(scale: torch.Tensor) -> torch.Tensor:
|
||||
)
|
||||
|
||||
return _upcast_e8m0_to_fp32(scale).contiguous()
|
||||
if scale.dtype == torch.uint8:
|
||||
# MXFP8 parameters preserve E8M0 scales as their raw exponent bytes.
|
||||
# They are biased exponents, not numeric uint8 scale values.
|
||||
return torch.exp2(scale.to(torch.int16).to(torch.float32) - 127.0)
|
||||
return scale.to(torch.float32)
|
||||
|
||||
|
||||
@@ -1226,7 +1320,15 @@ def _get_cached_wo_a_bf16(
|
||||
)
|
||||
|
||||
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
|
||||
if wo_a_scale_param is not None:
|
||||
if wo_a_scale_param is None:
|
||||
# ModelOpt MXFP8 stores the multiplicative E8M0 scale without the
|
||||
# historical ``_inv`` suffix.
|
||||
wo_a_scale_param = getattr(wo_a, "weight_scale", None)
|
||||
# Emulated MXFP8 kernels can replace the original one-byte weight with an
|
||||
# already-dequantized BF16 tensor while retaining the scale attribute for
|
||||
# metadata. Applying that retained scale again would double-dequantize the
|
||||
# weight. Block scaling is only valid while the one-byte FP8 storage remains.
|
||||
if wo_a_scale_param is not None and wo_a.weight.element_size() == 1:
|
||||
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
|
||||
torch.float32
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
@@ -2013,15 +2014,22 @@ def _get_packed_kv_cache_groups(
|
||||
cdiv(len(names), n) * page for page, names in page_size_layers.items()
|
||||
)
|
||||
|
||||
# Bytes a block must hold however the mamba buckets end up split: a mamba
|
||||
# bucket can go down to one state per group, every other bucket's split is
|
||||
# already fixed by the repeat pattern.
|
||||
# Bytes a block must hold however the state buckets end up split: mamba,
|
||||
# circular-buffer and unsplit sliding-window buckets can go down to one
|
||||
# state per group, every other bucket's split is fixed by the repeat pattern.
|
||||
def is_state_bucket(spec: UniformTypeKVCacheSpecs) -> bool:
|
||||
if isinstance(spec.first_spec, (MambaSpec, CircularBufferSpec)):
|
||||
return True
|
||||
return repeats_per_group is None and isinstance(
|
||||
spec.first_spec, SlidingWindowSpec
|
||||
)
|
||||
|
||||
anchor_bytes = max(
|
||||
(
|
||||
widest_group_bytes(
|
||||
page_size_layers,
|
||||
len(spec.kv_cache_specs)
|
||||
if isinstance(spec.first_spec, MambaSpec)
|
||||
if is_state_bucket(spec)
|
||||
else num_groups_for(spec, balanced),
|
||||
)
|
||||
for spec, page_size_layers, balanced in bucketed
|
||||
@@ -2032,10 +2040,9 @@ def _get_packed_kv_cache_groups(
|
||||
groups = []
|
||||
for spec, page_size_layers, balanced in bucketed:
|
||||
num_groups = num_groups_for(spec, balanced)
|
||||
# `_align_hybrid_block_size` pads a mamba state up to one attention
|
||||
# page, so cap a mamba group at the states a block already fits rather
|
||||
# than let it widen the block.
|
||||
if anchor_bytes and isinstance(spec.first_spec, MambaSpec):
|
||||
# Cap a state group at the states a block already fits rather than let
|
||||
# it widen the block.
|
||||
if anchor_bytes and is_state_bucket(spec):
|
||||
states_per_block = max(anchor_bytes // spec.first_spec.page_size_bytes, 1)
|
||||
num_groups = max(
|
||||
num_groups, cdiv(len(spec.kv_cache_specs), states_per_block)
|
||||
@@ -2071,8 +2078,9 @@ def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool:
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return False
|
||||
model_config = vllm_config.model_config
|
||||
return (
|
||||
model_config is not None and model_config.hf_config.model_type == "deepseek_v4"
|
||||
return model_config is not None and model_config.hf_config.model_type in (
|
||||
"deepseek_v4",
|
||||
"deepseek_v41",
|
||||
)
|
||||
|
||||
|
||||
@@ -2093,9 +2101,9 @@ def _annotate_eagle_groups(
|
||||
spec merging, wherever grouping happens to land. It is sufficient but
|
||||
not necessary: a drafter whose spec is indistinguishable from the
|
||||
target's cannot be found this way.
|
||||
2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses
|
||||
the target's own decoder layer and so carries no spec marker. Its draft
|
||||
attention layer is always the last registered layer, so flag whichever
|
||||
2. Model-scoped positional fallback for DeepseekV4/V4.1, whose MTP block
|
||||
reuses the target's own decoder layer and so carries no spec marker. Its
|
||||
draft attention layer is always the last registered layer, so flag whichever
|
||||
group holds it. This rule is only valid where the groups partition
|
||||
exactly the layers of ``kv_cache_spec``, which is true on the packed
|
||||
grouping path and not in general; other callers must leave
|
||||
@@ -2108,7 +2116,8 @@ def _annotate_eagle_groups(
|
||||
kv_cache_spec: The kv cache spec of each attention layer, in layer
|
||||
registration order. Only read by rule 2.
|
||||
kv_cache_groups: Groups to annotate in place.
|
||||
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
|
||||
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed
|
||||
group.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||
|
||||
@@ -51,7 +51,7 @@ from vllm.v1.simple_kv_offload.metadata import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
from vllm.v1.core.kv_cache_utils import BlockHashList, KVCacheBlock
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -783,6 +783,18 @@ class SimpleCPUOffloadScheduler:
|
||||
req_ids = list(dict.fromkeys(req_ids))
|
||||
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta
|
||||
|
||||
def _cached_gpu_block(
|
||||
self, resolved_hashes: "BlockHashList", block_idx: int, group_id: int
|
||||
) -> "KVCacheBlock | None":
|
||||
"""Return the GPU block still cached under this group's block hash."""
|
||||
if block_idx >= len(resolved_hashes):
|
||||
return None
|
||||
assert self._gpu_block_pool is not None
|
||||
blocks = self._gpu_block_pool.get_cached_block(
|
||||
resolved_hashes[block_idx], [group_id]
|
||||
)
|
||||
return blocks[0] if blocks else None
|
||||
|
||||
def _select_eager_blocks_to_store(
|
||||
self,
|
||||
state: StoreRequestState,
|
||||
@@ -827,18 +839,25 @@ class SimpleCPUOffloadScheduler:
|
||||
# the middle of the request.
|
||||
group_size = self.group_block_sizes[g]
|
||||
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
|
||||
resolved_hashes = (
|
||||
resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, group_size
|
||||
)
|
||||
if block_meta is not None
|
||||
else None
|
||||
resolved_hashes = resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, group_size
|
||||
)
|
||||
curr_mm_idx = 0
|
||||
secondary_mm_idx = 0
|
||||
start = state.num_stored_blocks[g]
|
||||
for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start):
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null:
|
||||
# Sliding-window groups null pages that left the window
|
||||
# before the connector sees the block table, but the
|
||||
# retained prefix-cache tail stays hashed in the GPU free
|
||||
# queue; store it from there.
|
||||
recovered = self._cached_gpu_block(resolved_hashes, i, g)
|
||||
if recovered is None:
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
gpu_block = recovered
|
||||
gpu_block_id = gpu_block.block_id
|
||||
if (
|
||||
self._classify_store_candidate(gpu_block)
|
||||
is not _StoreAdmission.READY
|
||||
@@ -854,7 +873,6 @@ class SimpleCPUOffloadScheduler:
|
||||
if block_meta is not None:
|
||||
token_start = i * group_size
|
||||
token_end = token_start + group_size
|
||||
assert resolved_hashes is not None
|
||||
parent_hash = (
|
||||
None
|
||||
if i == 0
|
||||
|
||||
@@ -17,9 +17,9 @@ def _resolve_dspark_attention_backend(
|
||||
) -> AttentionBackendEnum | None:
|
||||
if draft_backend is not None:
|
||||
return draft_backend
|
||||
# DeepSeek-V4 draft layers share the target's KV-cache layout. Other
|
||||
# DeepSeek-V4(.1) draft layers share the target's KV-cache layout. Other
|
||||
# DSpark architectures may use a different attention kind.
|
||||
if draft_model_config.hf_config.model_type == "deepseek_v4":
|
||||
if draft_model_config.hf_config.model_type in ("deepseek_v4", "deepseek_v41"):
|
||||
if target_backend is not None:
|
||||
logger.info_once(
|
||||
"Using the target model's %s attention backend for the "
|
||||
|
||||
@@ -112,7 +112,14 @@ def get_eagle3_aux_layers_from_config(
|
||||
if not layer_ids:
|
||||
dspark_layer_ids = getattr(hf_config, "dspark_target_layer_ids", None)
|
||||
if dspark_layer_ids:
|
||||
layer_ids = [i + 1 for i in dspark_layer_ids]
|
||||
if getattr(hf_config, "model_type", None) == "deepseek_v41":
|
||||
# v4.1 reads the attention *inputs* of its target layers, and
|
||||
# the target model captures the entry stream of layer L when
|
||||
# idx+1 == L, so the ids are used as-is. (v4's ids are in
|
||||
# capture-after semantics and keep the +1.)
|
||||
layer_ids = list(dspark_layer_ids)
|
||||
else:
|
||||
layer_ids = [i + 1 for i in dspark_layer_ids]
|
||||
if not layer_ids:
|
||||
# Dense DSpark (e.g. Qwen3) also uses different aux layer semantics.
|
||||
target_layer_ids = getattr(hf_config, "target_layer_ids", None)
|
||||
|
||||
@@ -813,6 +813,9 @@ class GPUModelRunner(
|
||||
|
||||
# Persistent buffers for CUDA graphs.
|
||||
self.input_ids = self._make_buffer(self.max_num_tokens, dtype=torch.int32)
|
||||
# [max_num_reqs, depth] prompt token ids preceding each request's chunk
|
||||
# start, for models exposing `token_lookback_depth`. Sized after load.
|
||||
self.lookback_token_ids: CpuGpuBuffer | None = None
|
||||
self.positions = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
@@ -1065,9 +1068,36 @@ class GPUModelRunner(
|
||||
)
|
||||
return self._mamba_bufs
|
||||
|
||||
def _init_model_kwargs(self):
|
||||
def _prepare_lookback_token_ids(self, num_reqs: int) -> torch.Tensor:
|
||||
"""Gather, per request, the `depth` prompt token ids preceding its
|
||||
first scheduled token (column j is position start - 1 - j); -1 where
|
||||
the position is before the prompt or already past it. Generated
|
||||
positions are left to the model: under async scheduling the CPU token
|
||||
table holds placeholders for them."""
|
||||
buf = self.lookback_token_ids
|
||||
assert buf is not None
|
||||
buf.np.fill(-1)
|
||||
if num_reqs > 0:
|
||||
depth = buf.np.shape[1]
|
||||
starts = self.input_batch.num_computed_tokens_cpu[:num_reqs, None]
|
||||
pos = starts - np.arange(1, depth + 1)
|
||||
num_prompt = self.input_batch.num_prompt_tokens[:num_reqs, None]
|
||||
valid = (pos >= 0) & (pos < num_prompt)
|
||||
rows = np.arange(num_reqs)[:, None]
|
||||
ids = self.input_batch.token_ids_cpu[rows, np.clip(pos, 0, None)]
|
||||
buf.np[:num_reqs] = np.where(valid, ids, -1)
|
||||
return buf.copy_to_gpu()
|
||||
|
||||
def _init_model_kwargs(self, num_reqs: int | None = None):
|
||||
model_kwargs = dict[str, Any]()
|
||||
|
||||
if self.lookback_token_ids is not None:
|
||||
if num_reqs is None:
|
||||
num_reqs = self.input_batch.num_reqs
|
||||
model_kwargs["lookback_token_ids"] = self._prepare_lookback_token_ids(
|
||||
num_reqs
|
||||
)
|
||||
|
||||
if not self.is_pooling_model:
|
||||
return model_kwargs
|
||||
|
||||
@@ -5314,6 +5344,11 @@ class GPUModelRunner(
|
||||
self.model = model_loader.load_model(
|
||||
vllm_config=self.vllm_config, model_config=self.model_config
|
||||
)
|
||||
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
|
||||
if lookback_depth > 0:
|
||||
self.lookback_token_ids = self._make_buffer(
|
||||
self.max_num_reqs, lookback_depth, dtype=torch.int32
|
||||
)
|
||||
if self.lora_config:
|
||||
self.model = self.load_lora_model(
|
||||
self.model, self.vllm_config, self.device
|
||||
@@ -6076,7 +6111,7 @@ class GPUModelRunner(
|
||||
):
|
||||
# Make sure padding doesn't exceed max_num_tokens
|
||||
assert num_tokens_padded <= self.max_num_tokens
|
||||
model_kwargs = self._init_model_kwargs()
|
||||
model_kwargs = self._init_model_kwargs(num_reqs=0)
|
||||
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
|
||||
input_ids, inputs_embeds = self._prepare_mm_inputs(num_tokens_padded)
|
||||
|
||||
@@ -6087,7 +6122,7 @@ class GPUModelRunner(
|
||||
elif self.enable_prompt_embeds:
|
||||
input_ids = None
|
||||
inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded]
|
||||
model_kwargs = self._init_model_kwargs()
|
||||
model_kwargs = self._init_model_kwargs(num_reqs=0)
|
||||
else:
|
||||
input_ids = self.input_ids.gpu[:num_tokens_padded]
|
||||
inputs_embeds = None
|
||||
|
||||
Reference in New Issue
Block a user