[Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174)

Signed-off-by: wan-danfeng <wandanfeng0802@gmail.com>
Signed-off-by: Wonderful <wandanfeng0802@gmail.com>
Co-authored-by: Wan_DF <wonderful199082@126.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Wonderful
2026-07-02 01:34:20 -07:00
committed by GitHub
co-authored by Wan_DF Benjamin Chislett Benjamin Chislett
parent 8357226f4f
commit 3af8789559
8 changed files with 364 additions and 5 deletions
@@ -86,6 +86,7 @@ only apply to model-based methods such as `draft_model`, `mtp`, `eagle3`, and
| `parallel_drafting` | `boolean` | `false` | Enable parallel draft token generation. Only compatible with EAGLE and draft-model methods. |
| `rejection_sample_method` | `string` | `strict` | `strict`, `probabilistic`, or `synthetic`. |
| `synthetic_acceptance_rate` | `float` | `None` | Average acceptance rate to target when `rejection_sample_method` is `synthetic`. Valid range is `[0, 1]`. |
| `use_heterogeneous_vocab` | `boolean` | `false` | Allow draft and target models with different vocabularies. Builds a token-level intersection at initialisation and constrains draft logits to shared tokens only. Only compatible with `method=draft_model`. Probabilistic draft sampling (`draft_sample_method='probabilistic'`) is not yet supported when this option is enabled. |
!!! note
Gemma 4 assistant checkpoints are handled as Gemma 4 MTP speculators, not
@@ -142,6 +143,33 @@ vllm serve <target-model> \
}'
```
#### Cross-Vocabulary Draft Models (TLI)
By default, vLLM requires the draft and target models to share the same
vocabulary. Setting `use_heterogeneous_vocab: true` enables the
**Token-Level Intersection (TLI)** algorithm, which allows draft models
from a different model family with a different tokenizer.
At initialisation, vLLM builds a mapping between the two vocabularies by
normalising token strings and computing their intersection. Draft logits are
constrained to the shared tokens before sampling, and the sampled token IDs
are translated to the target vocabulary before rejection sampling.
```python
from vllm import LLM, SamplingParams
llm = LLM(
model="Qwen/Qwen3-8B",
speculative_config={
"method": "draft_model",
"model": "HuggingFaceTB/SmolLM2-135M-Instruct",
"num_speculative_tokens": 3,
"use_heterogeneous_vocab": True,
},
gpu_memory_utilization=0.5,
)
```
### Notes
- `--speculative-config` expects a JSON object on the CLI. In YAML config
@@ -153,6 +181,7 @@ vllm serve <target-model> \
- Internal fields such as `target_model_config`, `draft_model_config`,
`target_parallel_config`, `draft_parallel_config`, and `draft_load_config`
are populated by vLLM and are not intended to be set by users.
- `use_heterogeneous_vocab` currently supports greedy draft sampling only. Probabilistic acceptance (temperature > 0 draft sampling) is not yet supported and will be added in a future release.
## Lossless guarantees of Speculative Decoding
@@ -76,6 +76,34 @@ The code used to request as completions as a client remains unchanged:
print(completion)
```
## Draft Model Method with heterogeneous vocabs
By default, vLLM requires the draft and target models to share the same vocabulary. Setting `use_heterogeneous_vocab: true` enables the **Token-Level Intersection (TLI)** algorithm, which allows draft models from a different model family with a different tokenizer.
Currently,`use_heterogeneous_vocab` currently requires `draft_sample_method='greedy'` (the default). Probabilistic draft sampling is not yet supported and will be added in a
future release.
```python
from vllm import LLM, SamplingParams
llm = LLM(
model="Qwen/Qwen3-8B",
speculative_config={
"method": "draft_model",
"model": "HuggingFaceTB/SmolLM2-135M-Instruct",
"num_speculative_tokens": 3,
"use_heterogeneous_vocab": True,
},
gpu_memory_utilization=0.5,
)
outputs = llm.generate(prompts,sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
```
!!! warning
Note: Please use `--speculative-config` to set all configurations related
to speculative decoding. The previous method of specifying the model
@@ -72,6 +72,7 @@ def parse_args():
parser.add_argument("--max-num-seqs", type=int, default=None)
parser.add_argument("--parallel-drafting", action="store_true")
parser.add_argument("--allowed-local-media-path", type=str, default="")
parser.add_argument("--use-heterogeneous-vocab", action="store_true")
return parser.parse_args()
@@ -135,6 +136,7 @@ def main(args):
"enforce_eager": args.enforce_eager,
"max_model_len": args.max_model_len,
"parallel_drafting": args.parallel_drafting,
"use_heterogeneous_vocab": args.use_heterogeneous_vocab,
}
elif args.method == "mtp":
speculative_config = {
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from vllm.v1.spec_decode.vocab_mapping import _detect_space_prefix
@pytest.mark.parametrize(
"model_name,expected_prefix",
[
# BPE tokenizer (GPT-2 family) uses Ġ (U+0120)
("HuggingFaceTB/SmolLM2-135M-Instruct", ("Ġ",)),
# SentencePiece tokenizer (LLaMA family) uses ▁ (U+2581)
("TinyLlama/TinyLlama-1.1B-Chat-v1.0", ("▁",)),
# BPE tokenizer (Qwen family) uses Ġ (U+0120)
("Qwen/Qwen2.5-0.5B-Instruct", ("Ġ",)),
],
)
def test_detect_space_prefix_real_tokenizers(model_name, expected_prefix):
tokenizer = AutoTokenizer.from_pretrained(model_name)
result = _detect_space_prefix(tokenizer)
assert result == expected_prefix, (
f"{model_name}: expected {expected_prefix!r}, got {result!r}"
)
def test_detect_space_prefix_fallback_on_failure():
"""When tokenizer lacks encode(), fall back to both known prefixes."""
class BrokenTokenizer:
def encode(self, text, **kwargs):
raise RuntimeError("broken")
result = _detect_space_prefix(BrokenTokenizer())
assert result == ("Ġ", "▁")
def test_detect_space_prefix_empty_encode():
"""When encode returns empty list, fall back."""
class EmptyTokenizer:
def encode(self, text, **kwargs):
return []
result = _detect_space_prefix(EmptyTokenizer())
assert result == ("Ġ", "▁")
+25 -2
View File
@@ -139,6 +139,12 @@ class SpeculativeConfig:
O(2 * tp_size) per token. Only applies to greedy draft selection in
non-tree speculation."""
use_heterogeneous_vocab: bool = False
"""Allow draft and target models to use different vocabularies.
When enabled, builds a token-level intersection at init and constrains
draft logits to shared tokens only (TLI algorithm). Requires
method='draft_model'."""
# Ngram proposer configuration
prompt_lookup_max: int | None = Field(default=None, ge=1)
"""Maximum size of ngram token window when using Ngram proposer, required
@@ -734,7 +740,11 @@ class SpeculativeConfig:
self.draft_model_config = ModelConfig(
model=self.model,
runner="draft",
tokenizer=self.target_model_config.tokenizer,
tokenizer=(
self.model
if self.use_heterogeneous_vocab
else self.target_model_config.tokenizer
),
tokenizer_mode=self.target_model_config.tokenizer_mode,
trust_remote_code=self.target_model_config.trust_remote_code,
allowed_local_media_path=self.target_model_config.allowed_local_media_path,
@@ -1103,7 +1113,20 @@ class SpeculativeConfig:
self.draft_parallel_config
)
self.verify_equal_vocab_size_if_draft_model()
if self.use_heterogeneous_vocab and not self.uses_draft_model():
raise ValueError(
"use_heterogeneous_vocab only works with method='draft_model'"
)
if self.use_heterogeneous_vocab and self.draft_sample_method != "greedy":
raise ValueError(
"use_heterogeneous_vocab currently only supports greedy draft "
"sampling. Set draft_sample_method='greedy' (the default) or "
"omit it."
)
if not self.use_heterogeneous_vocab:
self.verify_equal_vocab_size_if_draft_model()
return self
def verify_equal_vocab_size_if_draft_model(self):
+28 -1
View File
@@ -9,7 +9,9 @@ from vllm.config import VllmConfig
from vllm.config.utils import replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.tokenizers.registry import get_tokenizer
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.spec_decode.vocab_mapping import VocabMapping
logger = init_logger(__name__)
@@ -27,9 +29,34 @@ class DraftModelProposer(SpecDecodeBaseProposer):
pass_hidden_states_to_model=False,
runner=runner,
)
self._raise_if_vocab_size_mismatch()
self._raise_if_draft_tp_mismatch()
self.use_heterogeneous_vocab = self.speculative_config.use_heterogeneous_vocab
spec = self.speculative_config
if self.use_heterogeneous_vocab:
# Heterogeneous vocabularies: build a VocabMapping to translate
# token IDs between the two tokenizers and constrain draft logits
# to the intersection so rejection sampling stays lossless.
target_tokenizer = get_tokenizer(
spec.target_model_config.tokenizer,
trust_remote_code=spec.target_model_config.trust_remote_code,
)
draft_tokenizer = get_tokenizer(
spec.draft_model_config.model,
trust_remote_code=spec.draft_model_config.trust_remote_code,
)
self.vocab_mapping: VocabMapping | None = VocabMapping(
target_tokenizer=target_tokenizer,
draft_tokenizer=draft_tokenizer,
target_vocab_size=spec.target_model_config.get_vocab_size(),
draft_vocab_size=spec.draft_model_config.get_vocab_size(),
device=device,
)
else:
self._raise_if_vocab_size_mismatch()
self.vocab_mapping = None
def _raise_if_vocab_size_mismatch(self):
self.speculative_config.verify_equal_vocab_size_if_draft_model()
+50 -2
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from importlib.util import find_spec
from typing import Any, cast
from typing import TYPE_CHECKING, Any, cast
import numpy as np
import torch
@@ -15,6 +15,10 @@ from vllm.config import (
get_layers_from_vllm_config,
replace,
)
if TYPE_CHECKING:
from vllm.v1.spec_decode.vocab_mapping import VocabMapping
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.distributed.parallel_state import get_pp_group
from vllm.forward_context import set_forward_context
@@ -125,6 +129,11 @@ class SpecDecodeBaseProposer:
)
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
self.use_heterogeneous_vocab: bool = (
self.speculative_config.use_heterogeneous_vocab
)
self.vocab_mapping: VocabMapping | None = None
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
@@ -419,6 +428,12 @@ class SpecDecodeBaseProposer:
"""Greedy-sample draft tokens from hidden states."""
if self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
if self.use_heterogeneous_vocab:
logits = self.model.compute_logits(hidden_states)
assert self.vocab_mapping is not None
logits = self.vocab_mapping.constrain_draft_logits(logits)
draft_token_ids = logits.argmax(dim=-1)
return self.vocab_mapping.map_draft_to_target_ids(draft_token_ids)
return self.model.compute_logits(hidden_states).argmax(dim=-1)
def _sample_from_logits(
@@ -457,7 +472,28 @@ class SpecDecodeBaseProposer:
if not self._enable_probabilistic_draft_probs or sampling_metadata.all_greedy:
return self._greedy_sample(hidden_states), None
logits = self.model.compute_logits(hidden_states)
return self._sample_from_logits(logits, sampling_metadata)
if self.use_heterogeneous_vocab:
assert self.vocab_mapping is not None
logits = self.vocab_mapping.constrain_draft_logits(logits)
draft_token_ids, draft_probs = self._sample_from_logits(
logits, sampling_metadata
)
if self.use_heterogeneous_vocab:
assert self.vocab_mapping is not None
draft_token_ids = self.vocab_mapping.map_draft_to_target_ids(
draft_token_ids
)
# Config validation ensures draft_sample_method == "greedy" when
# use_heterogeneous_vocab is True, so this branch should never be
# reached. Kept as a safety fallback until probabilistic rejection
# sampling with heterogeneous vocabularies is implemented.
# TODO: remap draft_probs to target-vocab space for lossless
# probabilistic rejection sampling with heterogeneous vocabularies.
assert draft_probs is None, (
"probabilistic draft sampling is not supported with "
"use_heterogeneous_vocab"
)
return draft_token_ids, draft_probs
def take_last_draft_probs(self) -> torch.Tensor | None:
return self._last_draft_probs
@@ -647,6 +683,11 @@ class SpecDecodeBaseProposer:
# tensor.argmax() returns int64 by default.
input_ids = draft_token_ids_list[-1].int()
if self.use_heterogeneous_vocab:
# Map target token IDs to draft vocab space (TLI algorithm)
assert self.vocab_mapping is not None
input_ids = self.vocab_mapping.map_target_to_draft_ids(input_ids)
if not self.constant_draft_positions:
positions = self._update_positions_dependent_metadata(
positions,
@@ -785,6 +826,13 @@ class SpecDecodeBaseProposer:
cad: CommonAttentionMetadata,
num_rejected_tokens_gpu: torch.Tensor | None,
) -> tuple[int, torch.Tensor, CommonAttentionMetadata]:
# Map target token IDs to draft vocab space (TLI algorithm)
if self.use_heterogeneous_vocab:
assert self.vocab_mapping is not None
target_token_ids = self.vocab_mapping.map_target_to_draft_ids(
target_token_ids
)
next_token_ids = self.vocab_mapping.map_target_to_draft_ids(next_token_ids)
if not self.needs_extra_input_slots:
# Default EAGLE pathway: no reshaping of input tensors needed.
# Simply rotate the input ids and leave the positions unchanged,
+154
View File
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
def _detect_space_prefix(tokenizer) -> tuple[str, ...]:
"""Detect the space-prefix character(s) by tokenizing a literal space.
Different tokenizer families mark word-initial spaces differently:
BPE uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581). Probing at
runtime avoids hardcoding assumptions and correctly handles mixed-family
pairs (e.g. BPE draft + SentencePiece target).
"""
try:
space_ids = tokenizer.encode(" a", add_special_tokens=False)
if space_ids:
tok_str = tokenizer.convert_ids_to_tokens(space_ids[0])
if (
isinstance(tok_str, str)
and len(tok_str) > 1
and tok_str.endswith("a")
and tok_str[0] not in (" ", " ")
):
return (tok_str[:-1],)
except Exception:
pass
# Fallback: cover both BPE (Ġ U+0120) and SentencePiece (▁ U+2581)
return ("\u0120", "\u2581")
def _normalize_token(token: str, space_prefixes: tuple[str, ...]) -> str:
for prefix in space_prefixes:
if token.startswith(prefix):
return " " + token[len(prefix) :]
return token
def _get_unk_token_id(tokenizer, role: str) -> int:
"""Return a safe fallback token ID for out-of-intersection tokens.
Preferred: unk_token_id → eos_token_id → ValueError.
Checking with ``is not None`` is required because token ID 0 is a valid
(and common) unk ID on many tokenizers; using ``or 0`` would silently
mishandle those cases.
"""
unk = getattr(tokenizer, "unk_token_id", None)
if unk is not None:
return unk
eos = getattr(tokenizer, "eos_token_id", None)
if eos is not None:
logger.warning(
"VocabMapping: %s has no unk_token_id; "
"falling back to eos_token_id=%d for out-of-intersection tokens",
role,
eos,
)
return eos
raise ValueError(
f"VocabMapping: {role} has neither unk_token_id nor eos_token_id; "
"cannot safely map out-of-intersection tokens"
)
class VocabMapping:
def __init__(
self,
target_tokenizer,
draft_tokenizer,
target_vocab_size,
draft_vocab_size,
device,
):
self.target_vocab_size = target_vocab_size
self.draft_vocab_size = draft_vocab_size
self.device = device
self.target_unk_token_id = _get_unk_token_id(
target_tokenizer, "target tokenizer"
)
self.draft_unk_token_id = _get_unk_token_id(draft_tokenizer, "draft tokenizer")
target_prefixes = _detect_space_prefix(target_tokenizer)
draft_prefixes = _detect_space_prefix(draft_tokenizer)
target_vocab = target_tokenizer.get_vocab()
draft_vocab = draft_tokenizer.get_vocab()
target_normalized = {}
for token, tid in target_vocab.items():
norm = _normalize_token(token, target_prefixes)
if norm not in target_normalized:
target_normalized[norm] = tid
draft_normalized = {}
for token, tid in draft_vocab.items():
norm = _normalize_token(token, draft_prefixes)
if norm not in draft_normalized:
draft_normalized[norm] = tid
common_tokens = set(target_normalized.keys()) & set(draft_normalized.keys())
draft_to_target = torch.full((draft_vocab_size,), -1, dtype=torch.long)
target_to_draft = torch.full((target_vocab_size,), -1, dtype=torch.long)
intersection_mask_draft = torch.zeros(draft_vocab_size, dtype=torch.bool)
for norm_token in common_tokens:
t_id = target_normalized[norm_token]
d_id = draft_normalized[norm_token]
if t_id < target_vocab_size and d_id < draft_vocab_size:
draft_to_target[d_id] = t_id
target_to_draft[t_id] = d_id
intersection_mask_draft[d_id] = True
self.draft_to_target_ids = draft_to_target.to(device)
self.target_to_draft_ids = target_to_draft.to(device)
self.intersection_mask_draft = intersection_mask_draft.to(device)
self.intersection_size = int(intersection_mask_draft.sum().item())
logger.info(
"VocabMapping initialized: target_vocab=%d, draft_vocab=%d, "
"intersection=%d (%.1f%% of draft, %.1f%% of target)",
target_vocab_size,
draft_vocab_size,
self.intersection_size,
100.0 * self.intersection_size / max(draft_vocab_size, 1),
100.0 * self.intersection_size / max(target_vocab_size, 1),
)
if self.intersection_size < 100:
logger.warning(
"Very small vocabulary intersection (%d tokens).",
self.intersection_size,
)
def map_target_to_draft_ids(self, target_ids):
draft_ids = self.target_to_draft_ids[target_ids] # new tensor; no clone needed
missing = draft_ids == -1
if missing.any():
draft_ids[missing] = self.draft_unk_token_id
return draft_ids.to(target_ids.dtype)
def map_draft_to_target_ids(self, draft_ids):
target_ids = self.draft_to_target_ids[draft_ids] # new tensor; no clone needed
missing = target_ids == -1
if missing.any():
target_ids[missing] = self.target_unk_token_id
return target_ids.to(draft_ids.dtype)
def constrain_draft_logits(self, logits):
# masked_fill returns a new tensor; no clone needed
return logits.masked_fill(~self.intersection_mask_draft, float("-inf"))