[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -14,6 +14,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
)
|
||||
|
||||
return DFlashSpeculator(vllm_config, device)
|
||||
elif speculative_config.method == "dspark":
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import (
|
||||
DSparkSpeculator,
|
||||
)
|
||||
|
||||
return DSparkSpeculator(vllm_config, device)
|
||||
elif speculative_config.use_gemma4_mtp():
|
||||
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
|
||||
Gemma4Speculator,
|
||||
|
||||
@@ -29,6 +29,8 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DFlashSpeculator(DraftModelSpeculator):
|
||||
_speculator_name = "DFlash" # For logging, so we can share methods with subclasses
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
@@ -48,15 +50,17 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
self.dflash_causal = get_dflash_causal(self.draft_model_config)
|
||||
|
||||
# Buffers for context K/V precomputation. Populated by prepare_dflash_inputs,
|
||||
# and processed by the model's precompute_and_store_context_kv method.
|
||||
# NOT captured by CUDA graphs.
|
||||
# Whether the anchor query position is itself a prediction. DFlash default uses
|
||||
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
|
||||
# the anchor and the N-1 mask token positions. See _prepare_dflash_inputs_kernel
|
||||
self.sample_from_anchor = False
|
||||
|
||||
# Context positions for the K/V precompute. Populated by
|
||||
# prepare_dflash_inputs, and processed by the model's
|
||||
# precompute_and_store_context_kv method. NOT captured by CUDA graphs.
|
||||
self.context_positions = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.context_slot_mapping = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Per-mask-token sampling buffers. Flattened from (num_reqs, num_spec_tokens).
|
||||
max_num_sampled_tokens = self.max_num_reqs * self.num_speculative_steps
|
||||
@@ -94,7 +98,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
|
||||
def capture(self, attn_states: dict | None = None) -> None:
|
||||
logger.info("Capturing model for DFlash speculator...")
|
||||
logger.info("Capturing model for %s speculator...", self._speculator_name)
|
||||
# Reset sampling indices to zero to prevent stale values from prior
|
||||
# dummy runs from being baked into the captured graph.
|
||||
self.sample_indices.zero_()
|
||||
@@ -108,7 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
self.max_model_len,
|
||||
progress_bar_desc="Capturing dflash CUDA graphs",
|
||||
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
@@ -126,18 +130,39 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
) -> None:
|
||||
super().set_attn(model_state, kv_cache_config, block_tables)
|
||||
|
||||
# DFlash precomputes context K/V with a single block_size; mixing
|
||||
# kv-cache groups would silently corrupt the cache for the non-matching group.
|
||||
draft_groups = [gid for gid, g in enumerate(self.attn_groups) if g]
|
||||
assert len(draft_groups) == 1, (
|
||||
"DFlash currently requires all draft attention layers to share "
|
||||
"a single kv-cache group."
|
||||
)
|
||||
self.draft_kv_cache_group_id = draft_groups[0]
|
||||
self.draft_kv_cache_group_ids = [
|
||||
gid for gid, g in enumerate(self.attn_groups) if g
|
||||
]
|
||||
assert self.draft_kv_cache_group_ids, "No draft attention groups found."
|
||||
self.draft_kv_cache_group_id = self.draft_kv_cache_group_ids[0]
|
||||
self.draft_block_size = self.block_tables.block_sizes[
|
||||
self.draft_kv_cache_group_id
|
||||
]
|
||||
|
||||
# Per-group context slot buffers for the precompute (one row per group).
|
||||
self._context_slot_mappings = torch.zeros(
|
||||
len(self.draft_kv_cache_group_ids),
|
||||
self.max_num_tokens,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
# Map each draft decoder layer to the index (within draft_kv_cache_group_ids)
|
||||
# of the kv-cache group its cache belongs to. Models that share a single group
|
||||
# leave this as None and share one context slot mapping.
|
||||
self._layer_group_idx: list[int] | None = None
|
||||
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
||||
name_to_gid = {
|
||||
ln: gid
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
for ln in group.layer_names
|
||||
}
|
||||
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
|
||||
self._layer_group_idx = [
|
||||
gid_to_idx[name_to_gid[name]]
|
||||
for name in self.model.get_draft_kv_cache_layer_names()
|
||||
]
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
self,
|
||||
@@ -183,9 +208,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
num_sample = num_reqs * self.num_speculative_steps
|
||||
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
|
||||
# sample_pos is the predicted token's position Q; verification keys
|
||||
# Gumbel by the predecessor (Q-1). sample_draft adds +1, so pass Q-2.
|
||||
draft_tokens = self.sample_draft(
|
||||
sample_hidden_states,
|
||||
self.sample_pos[:num_sample],
|
||||
self.sample_pos[:num_sample] - 2,
|
||||
self.sample_idx_mapping[:num_sample],
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
@@ -294,40 +321,49 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# The query slot mapping is written into the shared BlockTables slot_mappings.
|
||||
# That buffer's address is what the captured CUDA graph reads from at replay.
|
||||
assert self.draft_kv_cache_group_id >= 0
|
||||
query_slot_mapping = self.block_tables.slot_mappings[
|
||||
self.draft_kv_cache_group_id
|
||||
]
|
||||
prepare_dflash_inputs(
|
||||
self.input_buffers,
|
||||
query_slot_mapping,
|
||||
self.context_positions,
|
||||
self.context_slot_mapping,
|
||||
self.sample_indices,
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
input_batch,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
self.block_tables.input_block_tables[self.draft_kv_cache_group_id],
|
||||
self.draft_block_size,
|
||||
self.parallel_drafting_token_id,
|
||||
self.num_query_per_req,
|
||||
self.num_speculative_steps,
|
||||
self.max_num_reqs,
|
||||
self.max_num_tokens,
|
||||
)
|
||||
# Support multiple draft KV cache groups by preparing inputs once for each
|
||||
for i, gid in enumerate(self.draft_kv_cache_group_ids):
|
||||
prepare_dflash_inputs(
|
||||
self.input_buffers,
|
||||
self.block_tables.slot_mappings[gid],
|
||||
self.context_positions,
|
||||
self._context_slot_mappings[i],
|
||||
self.sample_indices,
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
input_batch,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.block_sizes[gid],
|
||||
self.parallel_drafting_token_id,
|
||||
self.num_query_per_req,
|
||||
self.num_speculative_steps,
|
||||
self.max_num_reqs,
|
||||
self.max_num_tokens,
|
||||
self.max_model_len,
|
||||
self.sample_from_anchor,
|
||||
)
|
||||
|
||||
# Pre-insert context K/V into the cache. Runs eagerly outside the captured graph
|
||||
# because the context shape varies per step. During dummy runs the block tables
|
||||
# are placeholders, so we skip the cache write to avoid clobbering real entries.
|
||||
# Each layer uses the context slots of its own kv-cache group.
|
||||
if dummy_run:
|
||||
context_slots: torch.Tensor | list[torch.Tensor | None] | None = None
|
||||
elif self._layer_group_idx is not None:
|
||||
context_slots = [
|
||||
self._context_slot_mappings[gidx][:num_target_tokens]
|
||||
for gidx in self._layer_group_idx
|
||||
]
|
||||
else:
|
||||
context_slots = self._context_slot_mappings[0][:num_target_tokens]
|
||||
self.model.precompute_and_store_context_kv(
|
||||
self.hidden_states[:num_target_tokens],
|
||||
self.context_positions[:num_target_tokens],
|
||||
context_slot_mapping=(
|
||||
None if dummy_run else self.context_slot_mapping[:num_target_tokens]
|
||||
),
|
||||
context_slots,
|
||||
)
|
||||
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
@@ -408,6 +444,8 @@ def _prepare_dflash_inputs_kernel(
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
max_model_len,
|
||||
SAMPLE_FROM_ANCHOR: tl.constexpr,
|
||||
PAD_SLOT_ID: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
@@ -468,14 +506,21 @@ def _prepare_dflash_inputs_kernel(
|
||||
q_slot = q_block_id * block_size + (query_pos % block_size)
|
||||
|
||||
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
|
||||
tl.store(out_query_positions_ptr + query_idx, query_pos, mask=is_query)
|
||||
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
|
||||
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
|
||||
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
|
||||
|
||||
# --- Sample indices / positions / idx_mapping (mask tokens only) ---
|
||||
is_sample = is_query & (query_off > 0)
|
||||
sample_idx = req_idx * num_speculative_steps + (query_off - 1)
|
||||
# --- Sample indices / positions / idx_mapping ---
|
||||
# When SAMPLE_FROM_ANCHOR (DSpark), so we sample at EVERY query position
|
||||
# and each position k predicts the NEXT token (sampled position = query_pos + 1).
|
||||
# Otherwise (DFlash default) the anchor is the bonus token and only the mask tokens
|
||||
# at offsets > 0 are sampled from, each AT its own position.
|
||||
sample_off = 0 if SAMPLE_FROM_ANCHOR else 1
|
||||
is_sample = is_query & (query_off >= sample_off)
|
||||
sample_idx = req_idx * num_speculative_steps + (query_off - sample_off)
|
||||
sample_pos = query_pos + 1 if SAMPLE_FROM_ANCHOR else query_pos
|
||||
tl.store(out_sample_indices_ptr + sample_idx, query_idx, mask=is_sample)
|
||||
tl.store(out_sample_pos_ptr + sample_idx, query_pos, mask=is_sample)
|
||||
tl.store(out_sample_pos_ptr + sample_idx, sample_pos, mask=is_sample)
|
||||
tl.store(out_sample_idx_mapping_ptr + sample_idx, req_state_idx, mask=is_sample)
|
||||
|
||||
if block_idx == 0:
|
||||
@@ -542,6 +587,8 @@ def prepare_dflash_inputs(
|
||||
num_speculative_steps: int,
|
||||
max_num_reqs: int,
|
||||
max_num_tokens: int,
|
||||
max_model_len: int,
|
||||
sample_from_anchor: bool = False,
|
||||
) -> None:
|
||||
num_reqs = input_batch.num_reqs
|
||||
assert num_reqs > 0
|
||||
@@ -577,6 +624,8 @@ def prepare_dflash_inputs(
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
max_model_len,
|
||||
SAMPLE_FROM_ANCHOR=sample_from_anchor,
|
||||
PAD_SLOT_ID=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark speculator: semi-autoregressive parallel drafting.
|
||||
|
||||
DSpark drafts a block of ``num_speculative_tokens`` tokens in one parallel pass
|
||||
(reusing the DFlash machinery: context-KV precompute + a query-block forward),
|
||||
then injects intra-block dependency with a lightweight sequential Markov head.
|
||||
|
||||
Differences from DFlash:
|
||||
* Anchor-as-first-prediction: each request emits exactly ``N =
|
||||
num_speculative_tokens`` query tokens (anchor + N-1 noise), NOT ``1 + N``.
|
||||
Every query position is a prediction (the anchor predicts the first draft
|
||||
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
|
||||
(standard next-token), whereas DFlash's masks sit AT the predicted position.
|
||||
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
|
||||
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
|
||||
sample left-to-right, adding a prefix-dependent Markov bias derived from the
|
||||
previously sampled token at each step.
|
||||
|
||||
CUDA graphs (FULL, mirroring DFlash) cover the whole draft step: the parallel
|
||||
backbone forward AND the sequential Markov sampling.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model
|
||||
|
||||
|
||||
class DSparkSpeculator(DFlashSpeculator):
|
||||
_speculator_name = "DSpark"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
|
||||
self.num_query_per_req = self.num_speculative_steps
|
||||
|
||||
# DSpark consumes mean-pooled target aux hidden states at the target
|
||||
# layers, combined to hidden_size via main_proj. Store that combined
|
||||
# main_x (hidden_size wide). DSpark does not use the same pre-allocated buffer
|
||||
# that DeepSeek-V4's MTP uses.
|
||||
draft_hidden = self.draft_model_config.get_hidden_size()
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.dflash_causal = False
|
||||
|
||||
# The anchor query position is itself a prediction (see module docstring).
|
||||
self.sample_from_anchor = True
|
||||
|
||||
self._step_cols = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
self._anchor_idx = (
|
||||
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
* self.num_query_per_req
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: torch.nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> torch.nn.Module:
|
||||
return load_dspark_model(target_model, self.vllm_config)
|
||||
|
||||
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
# Sequential Markov sampling over the backbone's output hidden states.
|
||||
n_spec = self.num_speculative_steps
|
||||
num_sample = num_reqs * n_spec
|
||||
# Per-(req, position) head hidden, ordered (req, step).
|
||||
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
|
||||
base_logits = self.model.compute_logits(sample_hidden)
|
||||
vocab_size = base_logits.shape[-1]
|
||||
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
|
||||
|
||||
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
|
||||
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
|
||||
|
||||
# Anchor (bonus) token per request = the input id at query offset 0,
|
||||
# read via the precomputed persistent index (fixed buffer for capture).
|
||||
prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]]
|
||||
|
||||
for i in range(n_spec):
|
||||
# Sequential stage: Markov bias from the previously sampled token.
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
bias = self.model.markov_bias(markov_embed)
|
||||
logits_i = base_logits[:, i] + bias
|
||||
if self.draft_logits is not None:
|
||||
# sample_pos is the predicted token's position Q; the target
|
||||
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
|
||||
draft_i = gumbel_sample(
|
||||
logits_i,
|
||||
idx_map[:, i],
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
sample_pos[:, i] - 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=self.draft_logits,
|
||||
output_processed_logits_col=self._step_cols[i],
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
draft_i = logits_i.argmax(dim=-1)
|
||||
self.draft_tokens[:num_reqs, i] = draft_i
|
||||
prev = draft_i
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
# Full draft step (captured under CUDA graph): parallel backbone forward
|
||||
# then sequential Markov sampling over its hidden state outputs.
|
||||
head_hidden = self._run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
self._sample_sequential(num_reqs, head_hidden)
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
|
||||
|
||||
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
# DSpark uses non-causal attention.
|
||||
causal = False
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=not causal,
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
|
||||
with set_model_tag("dspark_head"):
|
||||
draft_model = get_model(
|
||||
vllm_config=draft_vllm_config, model_config=draft_model_config
|
||||
)
|
||||
|
||||
if get_pp_group().world_size != 1:
|
||||
raise NotImplementedError("DSpark does not support pipeline parallelism.")
|
||||
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
target_inner = target_language_model.model
|
||||
draft_inner = draft_model.model
|
||||
|
||||
target_embed = getattr(target_inner, "embed_tokens", None)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
draft_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
draft_lm_head = getattr(draft_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
):
|
||||
if draft_lm_head is not None:
|
||||
del draft_model.lm_head
|
||||
draft_model.lm_head = target_lm_head
|
||||
|
||||
return draft_model
|
||||
@@ -44,6 +44,15 @@ def get_eagle3_aux_layers_from_config(
|
||||
if dflash_config and isinstance(dflash_config, dict):
|
||||
# Add 1 to convert DFlash's aux layer id semantics
|
||||
layer_ids = [i + 1 for i in (dflash_config.get("target_layer_ids") or [])]
|
||||
if not layer_ids:
|
||||
dspark_layer_ids = getattr(hf_config, "dspark_target_layer_ids", None)
|
||||
if dspark_layer_ids:
|
||||
layer_ids = [i + 1 for i in dspark_layer_ids]
|
||||
if not layer_ids:
|
||||
# Dense DSpark (e.g. Qwen3) also uses different aux layer semantics.
|
||||
target_layer_ids = getattr(hf_config, "target_layer_ids", None)
|
||||
if target_layer_ids:
|
||||
layer_ids = [i + 1 for i in target_layer_ids]
|
||||
if layer_ids and isinstance(layer_ids, (list, tuple)):
|
||||
return tuple(layer_ids)
|
||||
return None
|
||||
|
||||
@@ -54,17 +54,23 @@ class DraftTokensHandler:
|
||||
def get_parallel_drafting_token_id(hf_config) -> int:
|
||||
"""Resolve the mask token id used for parallel drafting slots.
|
||||
|
||||
Checks (in order): `dflash_config.mask_token_id`, `pard_token`,
|
||||
`ptd_token_id`. Raises ValueError if none are present.
|
||||
Checks (in order): `dflash_config.mask_token_id`, top-level `mask_token_id`,
|
||||
`dspark_noise_token_id`, `pard_token`, `ptd_token_id`. Raises ValueError if
|
||||
none are present.
|
||||
"""
|
||||
dflash_config = getattr(hf_config, "dflash_config", None) or {}
|
||||
if "mask_token_id" in dflash_config:
|
||||
return int(dflash_config["mask_token_id"])
|
||||
if getattr(hf_config, "mask_token_id", None) is not None:
|
||||
return int(hf_config.mask_token_id)
|
||||
if hasattr(hf_config, "dspark_noise_token_id"):
|
||||
return int(hf_config.dspark_noise_token_id)
|
||||
if hasattr(hf_config, "pard_token"):
|
||||
return int(hf_config.pard_token)
|
||||
if hasattr(hf_config, "ptd_token_id"):
|
||||
return int(hf_config.ptd_token_id)
|
||||
raise ValueError(
|
||||
"Model config must specify `dflash_config.mask_token_id`,"
|
||||
" `pard_token`, or `ptd_token_id` for parallel drafting."
|
||||
" `mask_token_id`, `dspark_noise_token_id`, `pard_token`, or"
|
||||
" `ptd_token_id` for parallel drafting."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user