Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
+3 -1
View File
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -690,7 +691,8 @@ class ModelCudaGraphManager(CudaGraphManager):
}
if not self.is_first_pp_rank:
# Update for non-first PP ranks.
model_inputs["input_ids"] = None
if not requires_raw_input_tokens(model):
model_inputs["input_ids"] = None
model_inputs["inputs_embeds"] = None
assert intermediate_tensors is not None
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
+1 -3
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
from dataclasses import dataclass, replace
import torch
import torch.distributed as dist
from vllm.config import ParallelConfig
from vllm.config.compilation import CUDAGraphMode
@@ -63,7 +62,6 @@ def sync_cudagraph_and_dp_padding(
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
"""
assert dp_size > 1, "DP size must be greater than 1"
group = get_dp_group().cpu_group
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
tensor[0][dp_rank] = num_tokens
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
@@ -74,7 +72,7 @@ def sync_cudagraph_and_dp_padding(
if should_skip_dp_coordination():
tensor[:] = tensor[:, dp_rank, None].clone()
else:
dist.all_reduce(tensor, group=group)
get_dp_group().cpu_all_reduce_(tensor)
num_tokens_across_dp = tensor[0]
cg_mode_across_dp = tensor[1]
+15 -2
View File
@@ -33,7 +33,7 @@ import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.compilation.wrapper import compile_model_with_stock_torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.distributed.aux_output_connector.worker import (
AuxOutputWorkerConnector,
@@ -46,6 +46,9 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_ma
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
from vllm.model_executor.layers.quantization.online.lm_head import (
refresh_quantized_lm_heads,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.reload.inplace import reload_weights
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
@@ -569,6 +572,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return None
return speculator.model
def reset_lora_state(self) -> None:
super().reset_lora_state()
# Called whenever the target's weights are replaced (reload, weight
# transfer), so draft weights derived from them must follow.
draft_model = self.get_draft_model()
if draft_model is not None:
with set_current_vllm_config(self.vllm_config):
refresh_quantized_lm_heads(draft_model)
def reload_weights(
self,
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None,
@@ -1970,7 +1982,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
}
if not self.is_first_pp_rank:
# Update for non-first PP ranks.
model_inputs["input_ids"] = None
if not requires_raw_input_tokens(self.model):
model_inputs["input_ids"] = None
model_inputs["inputs_embeds"] = None
# Prepare the intermediate tensors.
+8
View File
@@ -70,6 +70,12 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
return u
@triton.jit
def spec_step_seed(seed, step):
"""Mix a speculative step into the high bits of a request seed."""
return seed ^ (tl.cast(step, tl.int64) << 32)
@triton.jit
def _murmur3_rotl32(value, shift: tl.constexpr):
return (value << shift) | (value >> (32 - shift))
@@ -257,6 +263,8 @@ def gumbel_block_argmax(
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
pos = tl.load(pos_ptr + token_idx)
if logits_cache_ptr is not None:
seed = spec_step_seed(seed, col)
return gumbel_noised_argmax(
logits,
block,
+252 -6
View File
@@ -25,7 +25,7 @@ from vllm.v1.watermarking.prfs.philox import (
from vllm.v1.watermarking.prfs.philox import (
_UINT32_MASK as _UINT32_MASK_VALUE,
)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
if HAS_TRITON:
from triton.language import math as tl_math
@@ -530,8 +530,10 @@ def _philox_gumbel_from_logits(logits, word):
@triton.jit
def _gumbel_value(logits_ptr, output, mask):
def _gumbel_value(logits_ptr, output, mask, temperature):
logits = tl.load(logits_ptr, mask=mask, other=float("-inf")).to(tl.float32)
if temperature is not None:
logits = tl_math.div_rn(logits, temperature)
return _philox_gumbel_from_logits(logits, output)
@@ -606,6 +608,7 @@ def _philox_gumbel_kernel(
USE_FP64: tl.constexpr,
IS_DRAFTING: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
block_index = tl.program_id(1)
@@ -619,6 +622,13 @@ def _philox_gumbel_kernel(
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
valid_req = req_state_idx >= 0
row_temperature = None
if APPLY_TEMPERATURE:
row_temperature = tl.load(
temp_ptr + req_state_idx, mask=valid_req, other=0.0
).to(tl.float32)
row_temperature = tl.where(row_temperature == 0, 1.0, row_temperature)
if logits_cache_ptr is not None:
if PER_TOKEN_COL:
col = tl.load(logits_cache_col_ptr + row)
@@ -644,10 +654,14 @@ def _philox_gumbel_kernel(
)
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
pos = tl.load(pos_ptr + row)
if logits_cache_ptr is not None:
seed = spec_step_seed(seed, col)
logits_row = logits_ptr + row * logits_stride
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
tl.float32
)
if row_temperature is not None:
logits = tl_math.div_rn(logits, row_temperature)
value, index = gumbel_noised_argmax(
logits,
candidate,
@@ -681,16 +695,16 @@ def _philox_gumbel_kernel(
candidate_3 = candidate_0 + 3
logits_row = logits_ptr + row * logits_stride
value_0 = _gumbel_value(
logits_row + candidate_0, output_0, candidate_0 < vocab_size
logits_row + candidate_0, output_0, candidate_0 < vocab_size, row_temperature
)
value_1 = _gumbel_value(
logits_row + candidate_1, output_1, candidate_1 < vocab_size
logits_row + candidate_1, output_1, candidate_1 < vocab_size, row_temperature
)
value_2 = _gumbel_value(
logits_row + candidate_2, output_2, candidate_2 < vocab_size
logits_row + candidate_2, output_2, candidate_2 < vocab_size, row_temperature
)
value_3 = _gumbel_value(
logits_row + candidate_3, output_3, candidate_3 < vocab_size
logits_row + candidate_3, output_3, candidate_3 < vocab_size, row_temperature
)
best_value = value_0
best_token = candidate_0
@@ -797,6 +811,238 @@ def philox_gumbel_sample(
USE_FP64=use_fp64,
IS_DRAFTING=is_drafting,
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
APPLY_TEMPERATURE=False,
)
max_block_index = local_max.argmax(dim=-1, keepdim=True)
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
@triton.jit(do_not_specialize=["step"])
def _draft_step_kernel(
sampled_ptr,
sampled_stride,
step,
local_argmax_ptr,
local_max_ptr,
num_blocks,
contexts_ptr,
context_stride,
skip_mask_ptr,
next_req_indices_ptr,
next_steps_ptr,
temp_ptr,
enabled_ptr,
all_token_ids_ptr,
all_token_ids_stride,
prompt_lens_ptr,
total_lens_ptr,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
CONTEXT_WIDTH: tl.constexpr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
MAX_HISTORY: tl.constexpr,
INCLUDE_PROMPT: tl.constexpr,
DEDUPLICATE: tl.constexpr,
SAMPLE: tl.constexpr,
PREPARE_NEXT: tl.constexpr,
BLOCKS_POW2: tl.constexpr,
):
"""Finish draft step `step` (if SAMPLE) and build the next step's skip mask."""
row = tl.program_id(0).to(tl.int64)
contexts_row_ptr = contexts_ptr + row * context_stride
if SAMPLE:
# Same pick as local_max.argmax(dim=-1): the first NaN, else the first max.
blocks = tl.arange(0, BLOCKS_POW2)
values = tl.load(
local_max_ptr + row * num_blocks + blocks,
mask=blocks < num_blocks,
other=float("-inf"),
)
is_nan = values != values
best = tl.max(tl.where(is_nan, float("-inf"), values), axis=0)
first_nan = tl.min(tl.where(is_nan, blocks, BLOCKS_POW2), axis=0)
first_max = tl.min(
tl.where((values == best) & (blocks < num_blocks), blocks, BLOCKS_POW2),
axis=0,
)
block = tl.where(first_nan < BLOCKS_POW2, first_nan, first_max)
token = tl.load(local_argmax_ptr + row * num_blocks + block)
tl.store(sampled_ptr + row * sampled_stride + step, token)
offsets = tl.arange(0, BLOCKS_POW2)
context = tl.load(
contexts_row_ptr + offsets + 1,
mask=offsets < CONTEXT_WIDTH - 1,
other=token,
)
tl.debug_barrier()
tl.store(contexts_row_ptr + offsets, context, mask=offsets < CONTEXT_WIDTH)
tl.debug_barrier()
if PREPARE_NEXT:
req_idx = tl.load(next_req_indices_ptr + row).to(tl.int64)
temperature = tl.load(temp_ptr + req_idx, mask=req_idx >= 0, other=0.0)
active = tl.load(enabled_ptr + row) & (temperature != 0)
if DEDUPLICATE:
# The dedup kernel body runs in place on the skip mask buffer.
tl.store(skip_mask_ptr + row, active)
tl.debug_barrier()
_repeated_context_mask_kernel(
skip_mask_ptr,
all_token_ids_ptr,
all_token_ids_stride,
next_req_indices_ptr,
prompt_lens_ptr,
total_lens_ptr,
next_steps_ptr,
1,
contexts_ptr,
context_stride,
None,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
next_steps_ptr,
1,
skip_mask_ptr,
CONTEXT_WIDTH=CONTEXT_WIDTH,
NUM_SPECULATIVE_STEPS=NUM_SPECULATIVE_STEPS,
MAX_HISTORY=MAX_HISTORY,
INCLUDE_PROMPT=INCLUDE_PROMPT,
SKIP_PARTIAL_CONTEXT=False,
HAS_HISTORY_OFFSETS=True,
SPECULATIVE_CONTEXTS=False,
DRAFT_CONTEXTS=True,
BLOCK=512,
)
tl.debug_barrier()
active = tl.load(skip_mask_ptr + row)
tl.store(skip_mask_ptr + row, active == 0)
def draft_philox_gumbel_sample(
logits: torch.Tensor,
contexts: torch.Tensor,
key: int,
*,
num_steps: int,
expanded_idx_mapping: torch.Tensor,
temperatures: torch.Tensor,
seeds: torch.Tensor,
positions: torch.Tensor,
enabled: torch.Tensor,
logits_cache: torch.Tensor,
logits_cache_col: torch.Tensor,
use_fp64: bool,
prior_contexts: torch.Tensor,
all_token_ids: torch.Tensor | None,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor | None,
deduplicate: bool,
max_history: int | None,
include_prompt: bool,
) -> torch.Tensor:
"""Sample `num_steps` draft steps flattened as (request, step), in step order.
Bit-exact with sampling each step by `DraftWatermarker.sample`, but with two
kernel launches per step: the Philox block argmax (with the temperature
divide and the logits cache write), then a per-row kernel that picks the
token, shifts `contexts` and builds the next step's skip mask. Padding rows
(index -1) read temperature 0 and are never watermarked, so they may differ.
"""
num_rows, vocab_size = logits.shape
num_reqs = num_rows // num_steps
assert logits.stride(-1) == 1 and contexts.stride(-1) == 1
assert logits.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= vocab_size, (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({vocab_size}). Cached logits would be truncated."
)
assert not deduplicate or (all_token_ids is not None and total_lens is not None), (
"context deduplication requires all_token_ids and total_lens"
)
logits = logits.view(num_reqs, num_steps, vocab_size)
# Step-major copies, so each step reads contiguous per-request slices.
expanded_idx_mapping, positions, logits_cache_col = (
tensor.view(num_reqs, num_steps).t().contiguous()
for tensor in (expanded_idx_mapping, positions, logits_cache_col)
)
block_size = 1024
num_blocks = triton.cdiv(vocab_size, block_size)
local_argmax = logits.new_empty(num_reqs, num_blocks, dtype=torch.int64)
local_max = logits.new_empty(
num_reqs, num_blocks, dtype=torch.float64 if use_fp64 else torch.float32
)
skip_mask = torch.empty(num_reqs, dtype=torch.bool, device=logits.device)
sampled = logits.new_empty(num_reqs, num_steps, dtype=torch.int64)
def finish_step(step: int) -> None:
next_step = min(step + 1, num_steps - 1)
_draft_step_kernel[(num_reqs,)](
sampled,
sampled.stride(0),
step,
local_argmax,
local_max,
num_blocks,
contexts,
contexts.stride(0),
skip_mask,
expanded_idx_mapping[next_step],
logits_cache_col[next_step],
temperatures,
enabled,
all_token_ids,
0 if all_token_ids is None else all_token_ids.stride(0),
prompt_lens,
total_lens,
prior_contexts,
prior_contexts.stride(0),
prior_contexts.stride(1),
CONTEXT_WIDTH=contexts.shape[-1],
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
DEDUPLICATE=deduplicate,
SAMPLE=step >= 0,
PREPARE_NEXT=step + 1 < num_steps,
BLOCKS_POW2=triton.next_power_of_2(max(num_blocks, contexts.shape[-1])),
)
finish_step(-1)
for step in range(num_steps):
_philox_gumbel_kernel[(num_reqs, num_blocks)](
local_argmax,
local_argmax.stride(0),
local_max,
local_max.stride(0),
logits[:, step],
logits.stride(0),
contexts,
contexts.stride(0),
skip_mask,
expanded_idx_mapping[step],
seeds,
positions[step],
temperatures,
logits_cache,
logits_cache.stride(0),
logits_cache.stride(1),
logits_cache_col[step],
logits[:, step],
logits.stride(0),
key & _UINT32_MASK_VALUE,
key >> 32,
vocab_size,
CONTEXT_WIDTH=contexts.shape[-1],
BLOCK_SIZE=block_size,
USE_FP64=use_fp64,
IS_DRAFTING=True,
PER_TOKEN_COL=True,
APPLY_TEMPERATURE=True,
)
finish_step(step)
return sampled.view(-1)
@@ -366,6 +366,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.seeds,
self.sample_col[:num_sample],
self.draft_logits,
num_steps=self.num_speculative_steps,
)
self.draft_tokens[:num_reqs] = draft_tokens.view(
num_reqs, self.num_speculative_steps
@@ -8,7 +8,7 @@ import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
@@ -58,7 +58,7 @@ def _selector_walk_kernel(
scores,
candidates,
mask & valid,
seed,
spec_step_seed(seed, step),
sample_pos,
temperature if SAMPLE_PROBABILISTIC else 0.0,
IS_DRAFTING=True,
+14 -2
View File
@@ -6,6 +6,9 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
)
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models.utils import PPMissingLayer
@@ -106,9 +109,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
if target_lm_head is not None and _should_share(
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
):
draft_head = target_lm_head
if speculative_config.draft_lm_head_quantization is not None:
draft_head = quantized_lm_head_copy(
target_lm_head, speculative_config.draft_lm_head_quantization
)
if draft_lm_head is not None:
del eagle_model.lm_head
eagle_model.lm_head = target_lm_head
eagle_model.lm_head = draft_head
# MTP layers route logits through layer.shared_head.head, not
# eagle_model.lm_head, so the per-layer copies need fixing up too.
@@ -119,7 +127,11 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
sh = getattr(layer, "shared_head", None)
if sh is not None and hasattr(sh, "head"):
del sh.head
sh.head = target_lm_head
sh.head = draft_head
elif speculative_config.draft_lm_head_quantization is not None:
raise ValueError(
"draft_lm_head_quantization needs a drafter that shares the target lm_head."
)
# MTP shares topk_indices_buffer with the target model. We update
# every module in the draft that holds a buffer reference so that
@@ -3,7 +3,11 @@
import torch
from vllm.triton_utils import tl, tldevice, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
from vllm.v1.worker.gpu.sample.gumbel import (
gumbel_block_argmax,
spec_step_seed,
tl_rand32,
)
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
@@ -569,7 +573,8 @@ def _rejection_kernel(
verifying &= is_valid_draft
if verifying:
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
step_seed = spec_step_seed(seed, i) if USE_BLOCK_VERIFICATION else seed
u = tl_rand32(step_seed, pos, includes_zero=False)
if is_greedy:
# Greedy sampling. Accept IFF draft matches target argmax.
# NOTE: Target argmax is stored directly so that resampling
+3 -1
View File
@@ -3,6 +3,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import replace
from functools import partial
from typing import TYPE_CHECKING, Any
import numpy as np
@@ -446,6 +447,7 @@ class DraftModelSpeculator(BaseSpeculator):
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
spec_step_idx: int = 0,
num_steps: int = 1,
) -> torch.Tensor:
if draft_logits is None and self.use_local_argmax_reduction:
return self.get_draft_top_tokens(hidden_states, spec_step_idx)
@@ -455,7 +457,7 @@ class DraftModelSpeculator(BaseSpeculator):
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
else partial(self.draft_watermarker.sample, num_steps=num_steps)
)
sampled = sampler(
logits,