Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
+4
View File
@@ -103,6 +103,10 @@ class AttentionConfig:
candidate pool instead, with fp32 logits, and keeps the masked dense walk
for steps whose contexts are too short for the pool to pay."""
indexer_tp_shard_min_rows: int = Field(default=65536, ge=1)
"""Minimum total prefill rows for the sparse-attention indexer to split
them across TP ranks instead of scoring every row on each rank."""
hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
+3
View File
@@ -139,6 +139,7 @@ MoEBackend = Literal[
"aiter",
"aiter_mega_moe",
"aiter_triton_mxfp4_bf16",
"aiter_triton_mxfp4_mxfp4",
"flydsl",
"hpc",
"emulation",
@@ -329,6 +330,8 @@ class KernelConfig:
DeepSeek-V4 only)
- "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16
(moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250)
- "aiter_triton_mxfp4_mxfp4": Use the AITER Triton/Gluon MXFP4 W4A4
(moe_gemm_a4w4) MoE kernel (ROCm gfx1250)
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
- "rdna3": Use the fused RDNA3 W4A16 HIP kernel (ROCm gfx1100 only)
- "hpc": Use HPC kernels (FP8 and Hopper only)
+4 -4
View File
@@ -1534,8 +1534,8 @@ class ModelConfig:
parallel_config: ParallelConfig,
) -> None:
if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1:
if not current_platform.is_cuda():
raise ValueError("--enable-hc-sp currently requires CUDA")
if not current_platform.is_cuda_alike():
raise ValueError("--enable-hc-sp currently requires CUDA or ROCm")
if not self.registry.is_hc_sp_supported_model(self.architectures, self):
raise ValueError("--enable-hc-sp is not supported for this model")
if parallel_config.pipeline_parallel_size != 1:
@@ -1622,8 +1622,8 @@ class ModelConfig:
def get_inputs_embeds_size(self) -> int:
# The size of inputs_embeds is usually identical to the size
# of the hidden states, however there are exceptions, such as
# embedding models like CLIP and SigLIP
names = ("projection_dim", "projection_size")
# OPT's projected token embeddings and embedding models like CLIP/SigLIP.
names = ("word_embed_proj_dim", "projection_dim", "projection_size")
return getattr_iter(
self.hf_text_config, names, default_factory=self.get_hidden_size
)
+7 -6
View File
@@ -15,6 +15,7 @@ from torch.distributed import ProcessGroup, ReduceOp, Store
import vllm.envs as envs
from vllm.config.fault_tolerance import FaultToleranceConfig
from vllm.config.utils import config
from vllm.distributed.utils import cpu_all_reduce_
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_ports_list
@@ -183,7 +184,7 @@ class ParallelConfig:
"""Whether the deployed model is MoE (if known)."""
enable_hc_sp: bool = False
"""Enable sequence parallelism for HyperConnection (HC) computation.
Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1).
Currently supported by Qwen4Exp on CUDA/ROCm with TP>1 and PP=1 (ignored when TP=1).
Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag.
"""
enable_expert_parallel: bool = False
@@ -887,7 +888,7 @@ class ParallelConfig:
tensor = torch.tensor(
[int(has_unfinished), int(pending_pause)], dtype=torch.int32, device="cpu"
)
torch.distributed.all_reduce(tensor, op=ReduceOp.SUM, group=dp_group)
cpu_all_reduce_(tensor, dp_group)
dp_size = dp_group.size()
pause_count = tensor[1].item()
has_unfinished_global = tensor[0].item() > 0 or pause_count % dp_size != 0
@@ -979,11 +980,11 @@ class ParallelConfig:
"Elastic EP is not supported with pipeline parallelism "
f"(pipeline_parallel_size={self.pipeline_parallel_size})."
)
if self.data_parallel_external_lb or self.data_parallel_hybrid_lb:
if self.data_parallel_hybrid_lb:
raise NotImplementedError(
"Elastic EP is not compatible with data_parallel_external_lb "
"or data_parallel_hybrid_lb. Elastic EP relies on a single API "
"server and core client to coordinate scale up/down."
"Elastic EP is not compatible with data_parallel_hybrid_lb. "
"Elastic EP supports a single API server/core client path "
"or data_parallel_external_lb, but not hybrid load balancing."
)
if self.eplb_config.use_async:
from vllm.distributed.nixl_utils import is_nixl_available
+6 -3
View File
@@ -201,9 +201,12 @@ class SchedulerConfig:
(the default) disables the watermark."""
prefill_schedule_interval: int = Field(default=1, ge=1)
"""For data-parallel deployments, only admit new prefill requests
once every N engine steps, aligned across DP ranks, to better balance
per-step forward-pass times."""
"""Only admit prefill once every N engine steps, leaving the steps in
between to decode alone. 1 (the default) admits prefill on every step.
A deferred step carries no prefill at all, including chunks of an
already-admitted prompt, so a long prompt can take up to N times as many
steps to finish prefilling."""
async_scheduling: bool | None = None
"""If set to False, disable async scheduling. Async scheduling helps to
+27 -5
View File
@@ -452,6 +452,12 @@ class SpeculativeConfig:
speculative methods. This is an experimental option for measuring the
acceptance-rate impact of reusing that block. It does not disable the
speculative drafter itself."""
draft_lm_head_quantization: Literal["fp8", "nvfp4"] | None = None
"""Quantize the drafter's copy of a target lm_head it shares (EAGLE and
MTP). "fp8" uses per-row weight and per-token activation scales on the
platform's FP8 linear kernel; "nvfp4" is weight-only on the platform's
W4A16 kernel. The target verifies with its own head, so only the
acceptance length can change. Model Runner V2 only."""
use_local_argmax_reduction: bool = False
"""Use vocab-parallel local argmax instead of all-gathering full logits
for draft token generation. Reduces communication from O(vocab_size) to
@@ -1780,6 +1786,20 @@ class SpeculativeConfig:
"n_predict parameter."
)
if self.draft_lm_head_quantization is not None:
# Gemma-4 MTP scores tied embedding rows directly; no lm_head to share.
if self.method not in ("eagle", "eagle3", "mtp") or self.use_gemma4_mtp():
raise ValueError(
"draft_lm_head_quantization is not supported for method "
f"{self.method!r}."
)
draft = self.draft_model_config
if draft is not None and draft.head_dtype != draft.dtype:
raise ValueError(
"draft_lm_head_quantization needs the lm_head to run in the "
f"model dtype, got head_dtype={draft.head_dtype}."
)
if self.num_speculative_tokens <= 0:
raise ValueError(
"Expected num_speculative_tokens to be greater "
@@ -1853,19 +1873,21 @@ class SpeculativeConfig:
==================== ============= ======== ================
EAGLE3 eagle3 No 0
P-EAGLE eagle3 Yes K - 1
DFlash dflash Yes K
DSpark dspark Yes K - 1
DFlash dflash Yes 0
DSpark dspark Yes 0
MTP mtp No 0
N-gram ngram No 0
Draft model draft_model No 1
PARD draft_model Yes K
==================== ============= ======== ================
DFlash and DSpark draft in a separate query pass rather than adding
rows to the target batch.
"""
num_draft_tokens = self.num_speculative_tokens
if self.use_dflash():
# DFlash uses one bonus query followed by K mask queries.
return num_draft_tokens
if self.use_dflash() or self.use_dspark():
return 0
if self.parallel_drafting:
if self.uses_draft_model():
+41 -9
View File
@@ -1325,15 +1325,26 @@ class VllmConfig:
)
if (
speculative_config.parallel_drafting
and speculative_config.method != "dspark"
and speculative_config.method not in ("dflash", "dspark")
):
raise ValueError(
"Parallel speculative drafting is not supported with watermarking."
)
if speculative_config.method not in ("dspark", "eagle", "eagle3", "mtp"):
if speculative_config.method not in (
"dflash",
"dspark",
"eagle",
"eagle3",
"mtp",
):
raise ValueError(
"Watermarking supports only autoregressive model-based "
"speculative decoding."
"Watermarking supports only dflash, dspark, eagle, eagle3 and "
"mtp speculative decoding."
)
if self._is_dflash_candidate_draft():
raise ValueError(
"Watermarking does not support DFlash2 and LiLiCorr "
"candidate-head drafters yet."
)
if (
not watermark_config.allow_target_only_watermarking
@@ -2307,9 +2318,6 @@ class VllmConfig:
worker is configured to handle.
"""
if self.speculative_config is not None:
scheduled_token_delta = (
self.speculative_config.max_num_new_slots_for_drafting
)
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
if self.scheduler_config.max_num_scheduled_tokens is None:
self.scheduler_config.max_num_scheduled_tokens = max_num_batched_tokens
@@ -2333,12 +2341,34 @@ class VllmConfig:
" num_speculative_tokens.",
)
if max_num_batched_tokens <= scheduled_token_delta:
# num_lookahead_tokens is at least every method's per-request draft
# slots, and covers the DFlash/DSpark draft pass's query rows.
if max_num_batched_tokens <= self.num_lookahead_tokens:
raise ValueError(
"VllmConfig does not have enough slots to schedule a token and"
" support the speculative decoding settings."
f" Got {max_num_batched_tokens=} and {scheduled_token_delta=}."
f" Got {max_num_batched_tokens=} and"
f" num_lookahead_tokens={self.num_lookahead_tokens}."
)
spec = self.speculative_config
if spec.use_dflash() or spec.use_dspark():
max_num_active_seqs = (
self.scheduler_config.max_num_active_seqs
or self.scheduler_config.max_num_seqs
)
max_draft_seqs = max_num_batched_tokens // self.num_lookahead_tokens
if max_draft_seqs < max_num_active_seqs:
logger.warning_once(
"The %s draft pass needs %d query tokens per request within "
"max_num_batched_tokens=%d. Overriding max_num_active_seqs "
"from %d to %d.",
spec.method,
self.num_lookahead_tokens,
max_num_batched_tokens,
max_num_active_seqs,
max_draft_seqs,
)
self.scheduler_config.max_num_active_seqs = max_draft_seqs
def _set_cudagraph_sizes(self):
"""VLLM defines the default candidate list of batch sizes for CUDA graph
@@ -3025,6 +3055,8 @@ class VllmConfig:
unsupported.append("dspark speculative decoding")
if self.speculative_config.enable_adaptive_verification:
unsupported.append("adaptive draft verification")
if self.speculative_config.draft_lm_head_quantization is not None:
unsupported.append("draft lm_head quantization")
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only).
if self._dflash_needs_multi_kv_group():