Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -690,7 +691,8 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
assert intermediate_tensors is not None
|
||||
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
|
||||
|
||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -63,7 +62,6 @@ def sync_cudagraph_and_dp_padding(
|
||||
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
|
||||
"""
|
||||
assert dp_size > 1, "DP size must be greater than 1"
|
||||
group = get_dp_group().cpu_group
|
||||
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor[0][dp_rank] = num_tokens
|
||||
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
||||
@@ -74,7 +72,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
if should_skip_dp_coordination():
|
||||
tensor[:] = tensor[:, dp_rank, None].clone()
|
||||
else:
|
||||
dist.all_reduce(tensor, group=group)
|
||||
get_dp_group().cpu_all_reduce_(tensor)
|
||||
|
||||
num_tokens_across_dp = tensor[0]
|
||||
cg_mode_across_dp = tensor[1]
|
||||
|
||||
@@ -33,7 +33,7 @@ import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.compilation.wrapper import compile_model_with_stock_torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
@@ -46,6 +46,9 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_ma
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
refresh_quantized_lm_heads,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.reload.inplace import reload_weights
|
||||
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
|
||||
@@ -569,6 +572,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return None
|
||||
return speculator.model
|
||||
|
||||
def reset_lora_state(self) -> None:
|
||||
super().reset_lora_state()
|
||||
# Called whenever the target's weights are replaced (reload, weight
|
||||
# transfer), so draft weights derived from them must follow.
|
||||
draft_model = self.get_draft_model()
|
||||
if draft_model is not None:
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
refresh_quantized_lm_heads(draft_model)
|
||||
|
||||
def reload_weights(
|
||||
self,
|
||||
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None,
|
||||
@@ -1970,7 +1982,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(self.model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
|
||||
# Prepare the intermediate tensors.
|
||||
|
||||
@@ -70,6 +70,12 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def spec_step_seed(seed, step):
|
||||
"""Mix a speculative step into the high bits of a request seed."""
|
||||
return seed ^ (tl.cast(step, tl.int64) << 32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_rotl32(value, shift: tl.constexpr):
|
||||
return (value << shift) | (value >> (32 - shift))
|
||||
@@ -257,6 +263,8 @@ def gumbel_block_argmax(
|
||||
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
if logits_cache_ptr is not None:
|
||||
seed = spec_step_seed(seed, col)
|
||||
return gumbel_noised_argmax(
|
||||
logits,
|
||||
block,
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.v1.watermarking.prfs.philox import (
|
||||
from vllm.v1.watermarking.prfs.philox import (
|
||||
_UINT32_MASK as _UINT32_MASK_VALUE,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
|
||||
|
||||
if HAS_TRITON:
|
||||
from triton.language import math as tl_math
|
||||
@@ -530,8 +530,10 @@ def _philox_gumbel_from_logits(logits, word):
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gumbel_value(logits_ptr, output, mask):
|
||||
def _gumbel_value(logits_ptr, output, mask, temperature):
|
||||
logits = tl.load(logits_ptr, mask=mask, other=float("-inf")).to(tl.float32)
|
||||
if temperature is not None:
|
||||
logits = tl_math.div_rn(logits, temperature)
|
||||
return _philox_gumbel_from_logits(logits, output)
|
||||
|
||||
|
||||
@@ -606,6 +608,7 @@ def _philox_gumbel_kernel(
|
||||
USE_FP64: tl.constexpr,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
block_index = tl.program_id(1)
|
||||
@@ -619,6 +622,13 @@ def _philox_gumbel_kernel(
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
|
||||
valid_req = req_state_idx >= 0
|
||||
|
||||
row_temperature = None
|
||||
if APPLY_TEMPERATURE:
|
||||
row_temperature = tl.load(
|
||||
temp_ptr + req_state_idx, mask=valid_req, other=0.0
|
||||
).to(tl.float32)
|
||||
row_temperature = tl.where(row_temperature == 0, 1.0, row_temperature)
|
||||
|
||||
if logits_cache_ptr is not None:
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(logits_cache_col_ptr + row)
|
||||
@@ -644,10 +654,14 @@ def _philox_gumbel_kernel(
|
||||
)
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + row)
|
||||
if logits_cache_ptr is not None:
|
||||
seed = spec_step_seed(seed, col)
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
|
||||
tl.float32
|
||||
)
|
||||
if row_temperature is not None:
|
||||
logits = tl_math.div_rn(logits, row_temperature)
|
||||
value, index = gumbel_noised_argmax(
|
||||
logits,
|
||||
candidate,
|
||||
@@ -681,16 +695,16 @@ def _philox_gumbel_kernel(
|
||||
candidate_3 = candidate_0 + 3
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
value_0 = _gumbel_value(
|
||||
logits_row + candidate_0, output_0, candidate_0 < vocab_size
|
||||
logits_row + candidate_0, output_0, candidate_0 < vocab_size, row_temperature
|
||||
)
|
||||
value_1 = _gumbel_value(
|
||||
logits_row + candidate_1, output_1, candidate_1 < vocab_size
|
||||
logits_row + candidate_1, output_1, candidate_1 < vocab_size, row_temperature
|
||||
)
|
||||
value_2 = _gumbel_value(
|
||||
logits_row + candidate_2, output_2, candidate_2 < vocab_size
|
||||
logits_row + candidate_2, output_2, candidate_2 < vocab_size, row_temperature
|
||||
)
|
||||
value_3 = _gumbel_value(
|
||||
logits_row + candidate_3, output_3, candidate_3 < vocab_size
|
||||
logits_row + candidate_3, output_3, candidate_3 < vocab_size, row_temperature
|
||||
)
|
||||
best_value = value_0
|
||||
best_token = candidate_0
|
||||
@@ -797,6 +811,238 @@ def philox_gumbel_sample(
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=is_drafting,
|
||||
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
|
||||
APPLY_TEMPERATURE=False,
|
||||
)
|
||||
max_block_index = local_max.argmax(dim=-1, keepdim=True)
|
||||
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["step"])
|
||||
def _draft_step_kernel(
|
||||
sampled_ptr,
|
||||
sampled_stride,
|
||||
step,
|
||||
local_argmax_ptr,
|
||||
local_max_ptr,
|
||||
num_blocks,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
skip_mask_ptr,
|
||||
next_req_indices_ptr,
|
||||
next_steps_ptr,
|
||||
temp_ptr,
|
||||
enabled_ptr,
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
MAX_HISTORY: tl.constexpr,
|
||||
INCLUDE_PROMPT: tl.constexpr,
|
||||
DEDUPLICATE: tl.constexpr,
|
||||
SAMPLE: tl.constexpr,
|
||||
PREPARE_NEXT: tl.constexpr,
|
||||
BLOCKS_POW2: tl.constexpr,
|
||||
):
|
||||
"""Finish draft step `step` (if SAMPLE) and build the next step's skip mask."""
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
contexts_row_ptr = contexts_ptr + row * context_stride
|
||||
if SAMPLE:
|
||||
# Same pick as local_max.argmax(dim=-1): the first NaN, else the first max.
|
||||
blocks = tl.arange(0, BLOCKS_POW2)
|
||||
values = tl.load(
|
||||
local_max_ptr + row * num_blocks + blocks,
|
||||
mask=blocks < num_blocks,
|
||||
other=float("-inf"),
|
||||
)
|
||||
is_nan = values != values
|
||||
best = tl.max(tl.where(is_nan, float("-inf"), values), axis=0)
|
||||
first_nan = tl.min(tl.where(is_nan, blocks, BLOCKS_POW2), axis=0)
|
||||
first_max = tl.min(
|
||||
tl.where((values == best) & (blocks < num_blocks), blocks, BLOCKS_POW2),
|
||||
axis=0,
|
||||
)
|
||||
block = tl.where(first_nan < BLOCKS_POW2, first_nan, first_max)
|
||||
token = tl.load(local_argmax_ptr + row * num_blocks + block)
|
||||
tl.store(sampled_ptr + row * sampled_stride + step, token)
|
||||
|
||||
offsets = tl.arange(0, BLOCKS_POW2)
|
||||
context = tl.load(
|
||||
contexts_row_ptr + offsets + 1,
|
||||
mask=offsets < CONTEXT_WIDTH - 1,
|
||||
other=token,
|
||||
)
|
||||
tl.debug_barrier()
|
||||
tl.store(contexts_row_ptr + offsets, context, mask=offsets < CONTEXT_WIDTH)
|
||||
tl.debug_barrier()
|
||||
|
||||
if PREPARE_NEXT:
|
||||
req_idx = tl.load(next_req_indices_ptr + row).to(tl.int64)
|
||||
temperature = tl.load(temp_ptr + req_idx, mask=req_idx >= 0, other=0.0)
|
||||
active = tl.load(enabled_ptr + row) & (temperature != 0)
|
||||
if DEDUPLICATE:
|
||||
# The dedup kernel body runs in place on the skip mask buffer.
|
||||
tl.store(skip_mask_ptr + row, active)
|
||||
tl.debug_barrier()
|
||||
_repeated_context_mask_kernel(
|
||||
skip_mask_ptr,
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
next_req_indices_ptr,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
next_steps_ptr,
|
||||
1,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
None,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
next_steps_ptr,
|
||||
1,
|
||||
skip_mask_ptr,
|
||||
CONTEXT_WIDTH=CONTEXT_WIDTH,
|
||||
NUM_SPECULATIVE_STEPS=NUM_SPECULATIVE_STEPS,
|
||||
MAX_HISTORY=MAX_HISTORY,
|
||||
INCLUDE_PROMPT=INCLUDE_PROMPT,
|
||||
SKIP_PARTIAL_CONTEXT=False,
|
||||
HAS_HISTORY_OFFSETS=True,
|
||||
SPECULATIVE_CONTEXTS=False,
|
||||
DRAFT_CONTEXTS=True,
|
||||
BLOCK=512,
|
||||
)
|
||||
tl.debug_barrier()
|
||||
active = tl.load(skip_mask_ptr + row)
|
||||
tl.store(skip_mask_ptr + row, active == 0)
|
||||
|
||||
|
||||
def draft_philox_gumbel_sample(
|
||||
logits: torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
key: int,
|
||||
*,
|
||||
num_steps: int,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
enabled: torch.Tensor,
|
||||
logits_cache: torch.Tensor,
|
||||
logits_cache_col: torch.Tensor,
|
||||
use_fp64: bool,
|
||||
prior_contexts: torch.Tensor,
|
||||
all_token_ids: torch.Tensor | None,
|
||||
prompt_lens: torch.Tensor,
|
||||
total_lens: torch.Tensor | None,
|
||||
deduplicate: bool,
|
||||
max_history: int | None,
|
||||
include_prompt: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Sample `num_steps` draft steps flattened as (request, step), in step order.
|
||||
|
||||
Bit-exact with sampling each step by `DraftWatermarker.sample`, but with two
|
||||
kernel launches per step: the Philox block argmax (with the temperature
|
||||
divide and the logits cache write), then a per-row kernel that picks the
|
||||
token, shifts `contexts` and builds the next step's skip mask. Padding rows
|
||||
(index -1) read temperature 0 and are never watermarked, so they may differ.
|
||||
"""
|
||||
num_rows, vocab_size = logits.shape
|
||||
num_reqs = num_rows // num_steps
|
||||
assert logits.stride(-1) == 1 and contexts.stride(-1) == 1
|
||||
assert logits.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= vocab_size, (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({vocab_size}). Cached logits would be truncated."
|
||||
)
|
||||
assert not deduplicate or (all_token_ids is not None and total_lens is not None), (
|
||||
"context deduplication requires all_token_ids and total_lens"
|
||||
)
|
||||
logits = logits.view(num_reqs, num_steps, vocab_size)
|
||||
# Step-major copies, so each step reads contiguous per-request slices.
|
||||
expanded_idx_mapping, positions, logits_cache_col = (
|
||||
tensor.view(num_reqs, num_steps).t().contiguous()
|
||||
for tensor in (expanded_idx_mapping, positions, logits_cache_col)
|
||||
)
|
||||
block_size = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, block_size)
|
||||
local_argmax = logits.new_empty(num_reqs, num_blocks, dtype=torch.int64)
|
||||
local_max = logits.new_empty(
|
||||
num_reqs, num_blocks, dtype=torch.float64 if use_fp64 else torch.float32
|
||||
)
|
||||
skip_mask = torch.empty(num_reqs, dtype=torch.bool, device=logits.device)
|
||||
sampled = logits.new_empty(num_reqs, num_steps, dtype=torch.int64)
|
||||
|
||||
def finish_step(step: int) -> None:
|
||||
next_step = min(step + 1, num_steps - 1)
|
||||
_draft_step_kernel[(num_reqs,)](
|
||||
sampled,
|
||||
sampled.stride(0),
|
||||
step,
|
||||
local_argmax,
|
||||
local_max,
|
||||
num_blocks,
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
skip_mask,
|
||||
expanded_idx_mapping[next_step],
|
||||
logits_cache_col[next_step],
|
||||
temperatures,
|
||||
enabled,
|
||||
all_token_ids,
|
||||
0 if all_token_ids is None else all_token_ids.stride(0),
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
prior_contexts,
|
||||
prior_contexts.stride(0),
|
||||
prior_contexts.stride(1),
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
DEDUPLICATE=deduplicate,
|
||||
SAMPLE=step >= 0,
|
||||
PREPARE_NEXT=step + 1 < num_steps,
|
||||
BLOCKS_POW2=triton.next_power_of_2(max(num_blocks, contexts.shape[-1])),
|
||||
)
|
||||
|
||||
finish_step(-1)
|
||||
for step in range(num_steps):
|
||||
_philox_gumbel_kernel[(num_reqs, num_blocks)](
|
||||
local_argmax,
|
||||
local_argmax.stride(0),
|
||||
local_max,
|
||||
local_max.stride(0),
|
||||
logits[:, step],
|
||||
logits.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
skip_mask,
|
||||
expanded_idx_mapping[step],
|
||||
seeds,
|
||||
positions[step],
|
||||
temperatures,
|
||||
logits_cache,
|
||||
logits_cache.stride(0),
|
||||
logits_cache.stride(1),
|
||||
logits_cache_col[step],
|
||||
logits[:, step],
|
||||
logits.stride(0),
|
||||
key & _UINT32_MASK_VALUE,
|
||||
key >> 32,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
BLOCK_SIZE=block_size,
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=True,
|
||||
PER_TOKEN_COL=True,
|
||||
APPLY_TEMPERATURE=True,
|
||||
)
|
||||
finish_step(step)
|
||||
return sampled.view(-1)
|
||||
|
||||
@@ -366,6 +366,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.seeds,
|
||||
self.sample_col[:num_sample],
|
||||
self.draft_logits,
|
||||
num_steps=self.num_speculative_steps,
|
||||
)
|
||||
self.draft_tokens[:num_reqs] = draft_tokens.view(
|
||||
num_reqs, self.num_speculative_steps
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ def _selector_walk_kernel(
|
||||
scores,
|
||||
candidates,
|
||||
mask & valid,
|
||||
seed,
|
||||
spec_step_seed(seed, step),
|
||||
sample_pos,
|
||||
temperature if SAMPLE_PROBABILISTIC else 0.0,
|
||||
IS_DRAFTING=True,
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
quantized_lm_head_copy,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
@@ -106,9 +109,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
if target_lm_head is not None and _should_share(
|
||||
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
):
|
||||
draft_head = target_lm_head
|
||||
if speculative_config.draft_lm_head_quantization is not None:
|
||||
draft_head = quantized_lm_head_copy(
|
||||
target_lm_head, speculative_config.draft_lm_head_quantization
|
||||
)
|
||||
if draft_lm_head is not None:
|
||||
del eagle_model.lm_head
|
||||
eagle_model.lm_head = target_lm_head
|
||||
eagle_model.lm_head = draft_head
|
||||
|
||||
# MTP layers route logits through layer.shared_head.head, not
|
||||
# eagle_model.lm_head, so the per-layer copies need fixing up too.
|
||||
@@ -119,7 +127,11 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
sh = getattr(layer, "shared_head", None)
|
||||
if sh is not None and hasattr(sh, "head"):
|
||||
del sh.head
|
||||
sh.head = target_lm_head
|
||||
sh.head = draft_head
|
||||
elif speculative_config.draft_lm_head_quantization is not None:
|
||||
raise ValueError(
|
||||
"draft_lm_head_quantization needs a drafter that shares the target lm_head."
|
||||
)
|
||||
|
||||
# MTP shares topk_indices_buffer with the target model. We update
|
||||
# every module in the draft that holds a buffer reference so that
|
||||
|
||||
@@ -3,7 +3,11 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
|
||||
from vllm.v1.worker.gpu.sample.gumbel import (
|
||||
gumbel_block_argmax,
|
||||
spec_step_seed,
|
||||
tl_rand32,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
|
||||
|
||||
|
||||
@@ -569,7 +573,8 @@ def _rejection_kernel(
|
||||
verifying &= is_valid_draft
|
||||
if verifying:
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand32(seed, pos, includes_zero=False)
|
||||
step_seed = spec_step_seed(seed, i) if USE_BLOCK_VERIFICATION else seed
|
||||
u = tl_rand32(step_seed, pos, includes_zero=False)
|
||||
if is_greedy:
|
||||
# Greedy sampling. Accept IFF draft matches target argmax.
|
||||
# NOTE: Target argmax is stored directly so that resampling
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
@@ -446,6 +447,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
spec_step_idx: int = 0,
|
||||
num_steps: int = 1,
|
||||
) -> torch.Tensor:
|
||||
if draft_logits is None and self.use_local_argmax_reduction:
|
||||
return self.get_draft_top_tokens(hidden_states, spec_step_idx)
|
||||
@@ -455,7 +457,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
sampler = (
|
||||
gumbel_sample
|
||||
if self.draft_watermarker is None
|
||||
else self.draft_watermarker.sample
|
||||
else partial(self.draft_watermarker.sample, num_steps=num_steps)
|
||||
)
|
||||
sampled = sampler(
|
||||
logits,
|
||||
|
||||
Reference in New Issue
Block a user