Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -103,6 +103,10 @@ class AttentionConfig:
|
||||
candidate pool instead, with fp32 logits, and keeps the masked dense walk
|
||||
for steps whose contexts are too short for the pool to pay."""
|
||||
|
||||
indexer_tp_shard_min_rows: int = Field(default=65536, ge=1)
|
||||
"""Minimum total prefill rows for the sparse-attention indexer to split
|
||||
them across TP ranks instead of scoring every row on each rank."""
|
||||
|
||||
hisparse_config: HiSparseConfig | None = None
|
||||
"""HiSparse host-resident KV configuration. Setting this enables experimental
|
||||
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
|
||||
|
||||
@@ -139,6 +139,7 @@ MoEBackend = Literal[
|
||||
"aiter",
|
||||
"aiter_mega_moe",
|
||||
"aiter_triton_mxfp4_bf16",
|
||||
"aiter_triton_mxfp4_mxfp4",
|
||||
"flydsl",
|
||||
"hpc",
|
||||
"emulation",
|
||||
@@ -329,6 +330,8 @@ class KernelConfig:
|
||||
DeepSeek-V4 only)
|
||||
- "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16
|
||||
(moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250)
|
||||
- "aiter_triton_mxfp4_mxfp4": Use the AITER Triton/Gluon MXFP4 W4A4
|
||||
(moe_gemm_a4w4) MoE kernel (ROCm gfx1250)
|
||||
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
|
||||
- "rdna3": Use the fused RDNA3 W4A16 HIP kernel (ROCm gfx1100 only)
|
||||
- "hpc": Use HPC kernels (FP8 and Hopper only)
|
||||
|
||||
@@ -1534,8 +1534,8 @@ class ModelConfig:
|
||||
parallel_config: ParallelConfig,
|
||||
) -> None:
|
||||
if parallel_config.enable_hc_sp and parallel_config.tensor_parallel_size > 1:
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("--enable-hc-sp currently requires CUDA")
|
||||
if not current_platform.is_cuda_alike():
|
||||
raise ValueError("--enable-hc-sp currently requires CUDA or ROCm")
|
||||
if not self.registry.is_hc_sp_supported_model(self.architectures, self):
|
||||
raise ValueError("--enable-hc-sp is not supported for this model")
|
||||
if parallel_config.pipeline_parallel_size != 1:
|
||||
@@ -1622,8 +1622,8 @@ class ModelConfig:
|
||||
def get_inputs_embeds_size(self) -> int:
|
||||
# The size of inputs_embeds is usually identical to the size
|
||||
# of the hidden states, however there are exceptions, such as
|
||||
# embedding models like CLIP and SigLIP
|
||||
names = ("projection_dim", "projection_size")
|
||||
# OPT's projected token embeddings and embedding models like CLIP/SigLIP.
|
||||
names = ("word_embed_proj_dim", "projection_dim", "projection_size")
|
||||
return getattr_iter(
|
||||
self.hf_text_config, names, default_factory=self.get_hidden_size
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ from torch.distributed import ProcessGroup, ReduceOp, Store
|
||||
import vllm.envs as envs
|
||||
from vllm.config.fault_tolerance import FaultToleranceConfig
|
||||
from vllm.config.utils import config
|
||||
from vllm.distributed.utils import cpu_all_reduce_
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
@@ -183,7 +184,7 @@ class ParallelConfig:
|
||||
"""Whether the deployed model is MoE (if known)."""
|
||||
enable_hc_sp: bool = False
|
||||
"""Enable sequence parallelism for HyperConnection (HC) computation.
|
||||
Currently supported by Qwen4Exp on CUDA with TP>1 and PP=1 (ignored when TP=1).
|
||||
Currently supported by Qwen4Exp on CUDA/ROCm with TP>1 and PP=1 (ignored when TP=1).
|
||||
Does not change MoE parallelism. MoE SP may enable HC SP independently of this flag.
|
||||
"""
|
||||
enable_expert_parallel: bool = False
|
||||
@@ -887,7 +888,7 @@ class ParallelConfig:
|
||||
tensor = torch.tensor(
|
||||
[int(has_unfinished), int(pending_pause)], dtype=torch.int32, device="cpu"
|
||||
)
|
||||
torch.distributed.all_reduce(tensor, op=ReduceOp.SUM, group=dp_group)
|
||||
cpu_all_reduce_(tensor, dp_group)
|
||||
dp_size = dp_group.size()
|
||||
pause_count = tensor[1].item()
|
||||
has_unfinished_global = tensor[0].item() > 0 or pause_count % dp_size != 0
|
||||
@@ -979,11 +980,11 @@ class ParallelConfig:
|
||||
"Elastic EP is not supported with pipeline parallelism "
|
||||
f"(pipeline_parallel_size={self.pipeline_parallel_size})."
|
||||
)
|
||||
if self.data_parallel_external_lb or self.data_parallel_hybrid_lb:
|
||||
if self.data_parallel_hybrid_lb:
|
||||
raise NotImplementedError(
|
||||
"Elastic EP is not compatible with data_parallel_external_lb "
|
||||
"or data_parallel_hybrid_lb. Elastic EP relies on a single API "
|
||||
"server and core client to coordinate scale up/down."
|
||||
"Elastic EP is not compatible with data_parallel_hybrid_lb. "
|
||||
"Elastic EP supports a single API server/core client path "
|
||||
"or data_parallel_external_lb, but not hybrid load balancing."
|
||||
)
|
||||
if self.eplb_config.use_async:
|
||||
from vllm.distributed.nixl_utils import is_nixl_available
|
||||
|
||||
@@ -201,9 +201,12 @@ class SchedulerConfig:
|
||||
(the default) disables the watermark."""
|
||||
|
||||
prefill_schedule_interval: int = Field(default=1, ge=1)
|
||||
"""For data-parallel deployments, only admit new prefill requests
|
||||
once every N engine steps, aligned across DP ranks, to better balance
|
||||
per-step forward-pass times."""
|
||||
"""Only admit prefill once every N engine steps, leaving the steps in
|
||||
between to decode alone. 1 (the default) admits prefill on every step.
|
||||
|
||||
A deferred step carries no prefill at all, including chunks of an
|
||||
already-admitted prompt, so a long prompt can take up to N times as many
|
||||
steps to finish prefilling."""
|
||||
|
||||
async_scheduling: bool | None = None
|
||||
"""If set to False, disable async scheduling. Async scheduling helps to
|
||||
|
||||
@@ -452,6 +452,12 @@ class SpeculativeConfig:
|
||||
speculative methods. This is an experimental option for measuring the
|
||||
acceptance-rate impact of reusing that block. It does not disable the
|
||||
speculative drafter itself."""
|
||||
draft_lm_head_quantization: Literal["fp8", "nvfp4"] | None = None
|
||||
"""Quantize the drafter's copy of a target lm_head it shares (EAGLE and
|
||||
MTP). "fp8" uses per-row weight and per-token activation scales on the
|
||||
platform's FP8 linear kernel; "nvfp4" is weight-only on the platform's
|
||||
W4A16 kernel. The target verifies with its own head, so only the
|
||||
acceptance length can change. Model Runner V2 only."""
|
||||
use_local_argmax_reduction: bool = False
|
||||
"""Use vocab-parallel local argmax instead of all-gathering full logits
|
||||
for draft token generation. Reduces communication from O(vocab_size) to
|
||||
@@ -1780,6 +1786,20 @@ class SpeculativeConfig:
|
||||
"n_predict parameter."
|
||||
)
|
||||
|
||||
if self.draft_lm_head_quantization is not None:
|
||||
# Gemma-4 MTP scores tied embedding rows directly; no lm_head to share.
|
||||
if self.method not in ("eagle", "eagle3", "mtp") or self.use_gemma4_mtp():
|
||||
raise ValueError(
|
||||
"draft_lm_head_quantization is not supported for method "
|
||||
f"{self.method!r}."
|
||||
)
|
||||
draft = self.draft_model_config
|
||||
if draft is not None and draft.head_dtype != draft.dtype:
|
||||
raise ValueError(
|
||||
"draft_lm_head_quantization needs the lm_head to run in the "
|
||||
f"model dtype, got head_dtype={draft.head_dtype}."
|
||||
)
|
||||
|
||||
if self.num_speculative_tokens <= 0:
|
||||
raise ValueError(
|
||||
"Expected num_speculative_tokens to be greater "
|
||||
@@ -1853,19 +1873,21 @@ class SpeculativeConfig:
|
||||
==================== ============= ======== ================
|
||||
EAGLE3 eagle3 No 0
|
||||
P-EAGLE eagle3 Yes K - 1
|
||||
DFlash dflash Yes K
|
||||
DSpark dspark Yes K - 1
|
||||
DFlash dflash Yes 0
|
||||
DSpark dspark Yes 0
|
||||
MTP mtp No 0
|
||||
N-gram ngram No 0
|
||||
Draft model draft_model No 1
|
||||
PARD draft_model Yes K
|
||||
==================== ============= ======== ================
|
||||
|
||||
DFlash and DSpark draft in a separate query pass rather than adding
|
||||
rows to the target batch.
|
||||
"""
|
||||
num_draft_tokens = self.num_speculative_tokens
|
||||
|
||||
if self.use_dflash():
|
||||
# DFlash uses one bonus query followed by K mask queries.
|
||||
return num_draft_tokens
|
||||
if self.use_dflash() or self.use_dspark():
|
||||
return 0
|
||||
|
||||
if self.parallel_drafting:
|
||||
if self.uses_draft_model():
|
||||
|
||||
+41
-9
@@ -1325,15 +1325,26 @@ class VllmConfig:
|
||||
)
|
||||
if (
|
||||
speculative_config.parallel_drafting
|
||||
and speculative_config.method != "dspark"
|
||||
and speculative_config.method not in ("dflash", "dspark")
|
||||
):
|
||||
raise ValueError(
|
||||
"Parallel speculative drafting is not supported with watermarking."
|
||||
)
|
||||
if speculative_config.method not in ("dspark", "eagle", "eagle3", "mtp"):
|
||||
if speculative_config.method not in (
|
||||
"dflash",
|
||||
"dspark",
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"mtp",
|
||||
):
|
||||
raise ValueError(
|
||||
"Watermarking supports only autoregressive model-based "
|
||||
"speculative decoding."
|
||||
"Watermarking supports only dflash, dspark, eagle, eagle3 and "
|
||||
"mtp speculative decoding."
|
||||
)
|
||||
if self._is_dflash_candidate_draft():
|
||||
raise ValueError(
|
||||
"Watermarking does not support DFlash2 and LiLiCorr "
|
||||
"candidate-head drafters yet."
|
||||
)
|
||||
if (
|
||||
not watermark_config.allow_target_only_watermarking
|
||||
@@ -2307,9 +2318,6 @@ class VllmConfig:
|
||||
worker is configured to handle.
|
||||
"""
|
||||
if self.speculative_config is not None:
|
||||
scheduled_token_delta = (
|
||||
self.speculative_config.max_num_new_slots_for_drafting
|
||||
)
|
||||
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
if self.scheduler_config.max_num_scheduled_tokens is None:
|
||||
self.scheduler_config.max_num_scheduled_tokens = max_num_batched_tokens
|
||||
@@ -2333,12 +2341,34 @@ class VllmConfig:
|
||||
" num_speculative_tokens.",
|
||||
)
|
||||
|
||||
if max_num_batched_tokens <= scheduled_token_delta:
|
||||
# num_lookahead_tokens is at least every method's per-request draft
|
||||
# slots, and covers the DFlash/DSpark draft pass's query rows.
|
||||
if max_num_batched_tokens <= self.num_lookahead_tokens:
|
||||
raise ValueError(
|
||||
"VllmConfig does not have enough slots to schedule a token and"
|
||||
" support the speculative decoding settings."
|
||||
f" Got {max_num_batched_tokens=} and {scheduled_token_delta=}."
|
||||
f" Got {max_num_batched_tokens=} and"
|
||||
f" num_lookahead_tokens={self.num_lookahead_tokens}."
|
||||
)
|
||||
spec = self.speculative_config
|
||||
if spec.use_dflash() or spec.use_dspark():
|
||||
max_num_active_seqs = (
|
||||
self.scheduler_config.max_num_active_seqs
|
||||
or self.scheduler_config.max_num_seqs
|
||||
)
|
||||
max_draft_seqs = max_num_batched_tokens // self.num_lookahead_tokens
|
||||
if max_draft_seqs < max_num_active_seqs:
|
||||
logger.warning_once(
|
||||
"The %s draft pass needs %d query tokens per request within "
|
||||
"max_num_batched_tokens=%d. Overriding max_num_active_seqs "
|
||||
"from %d to %d.",
|
||||
spec.method,
|
||||
self.num_lookahead_tokens,
|
||||
max_num_batched_tokens,
|
||||
max_num_active_seqs,
|
||||
max_draft_seqs,
|
||||
)
|
||||
self.scheduler_config.max_num_active_seqs = max_draft_seqs
|
||||
|
||||
def _set_cudagraph_sizes(self):
|
||||
"""VLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
@@ -3025,6 +3055,8 @@ class VllmConfig:
|
||||
unsupported.append("dspark speculative decoding")
|
||||
if self.speculative_config.enable_adaptive_verification:
|
||||
unsupported.append("adaptive draft verification")
|
||||
if self.speculative_config.draft_lm_head_quantization is not None:
|
||||
unsupported.append("draft lm_head quantization")
|
||||
|
||||
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only).
|
||||
if self._dflash_needs_multi_kv_group():
|
||||
|
||||
Reference in New Issue
Block a user