From 3af878955935dd356182f6dd2ea9660acb1757be Mon Sep 17 00:00:00 2001 From: Wonderful Date: Thu, 2 Jul 2026 16:34:20 +0800 Subject: [PATCH] [Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174) Signed-off-by: wan-danfeng Signed-off-by: Wonderful Co-authored-by: Wan_DF Co-authored-by: Benjamin Chislett Co-authored-by: Benjamin Chislett --- docs/features/speculative_decoding/README.md | 29 ++++ .../speculative_decoding/draft_model.md | 28 ++++ .../spec_decode_offline.py | 2 + tests/v1/spec_decode/test_vocab_mapping.py | 48 ++++++ vllm/config/speculative.py | 27 ++- vllm/v1/spec_decode/draft_model.py | 29 +++- vllm/v1/spec_decode/llm_base_proposer.py | 52 +++++- vllm/v1/spec_decode/vocab_mapping.py | 154 ++++++++++++++++++ 8 files changed, 364 insertions(+), 5 deletions(-) create mode 100644 tests/v1/spec_decode/test_vocab_mapping.py create mode 100644 vllm/v1/spec_decode/vocab_mapping.py diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index 65f396e04a..ceb25dbfd0 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -86,6 +86,7 @@ only apply to model-based methods such as `draft_model`, `mtp`, `eagle3`, and | `parallel_drafting` | `boolean` | `false` | Enable parallel draft token generation. Only compatible with EAGLE and draft-model methods. | | `rejection_sample_method` | `string` | `strict` | `strict`, `probabilistic`, or `synthetic`. | | `synthetic_acceptance_rate` | `float` | `None` | Average acceptance rate to target when `rejection_sample_method` is `synthetic`. Valid range is `[0, 1]`. | + | `use_heterogeneous_vocab` | `boolean` | `false` | Allow draft and target models with different vocabularies. Builds a token-level intersection at initialisation and constrains draft logits to shared tokens only. Only compatible with `method=draft_model`. Probabilistic draft sampling (`draft_sample_method='probabilistic'`) is not yet supported when this option is enabled. | !!! note Gemma 4 assistant checkpoints are handled as Gemma 4 MTP speculators, not @@ -142,6 +143,33 @@ vllm serve \ }' ``` +#### Cross-Vocabulary Draft Models (TLI) + + By default, vLLM requires the draft and target models to share the same + vocabulary. Setting `use_heterogeneous_vocab: true` enables the + **Token-Level Intersection (TLI)** algorithm, which allows draft models + from a different model family with a different tokenizer. + + At initialisation, vLLM builds a mapping between the two vocabularies by + normalising token strings and computing their intersection. Draft logits are + constrained to the shared tokens before sampling, and the sampled token IDs + are translated to the target vocabulary before rejection sampling. + + ```python + from vllm import LLM, SamplingParams + + llm = LLM( + model="Qwen/Qwen3-8B", + speculative_config={ + "method": "draft_model", + "model": "HuggingFaceTB/SmolLM2-135M-Instruct", + "num_speculative_tokens": 3, + "use_heterogeneous_vocab": True, + }, + gpu_memory_utilization=0.5, + ) +``` + ### Notes - `--speculative-config` expects a JSON object on the CLI. In YAML config @@ -153,6 +181,7 @@ vllm serve \ - Internal fields such as `target_model_config`, `draft_model_config`, `target_parallel_config`, `draft_parallel_config`, and `draft_load_config` are populated by vLLM and are not intended to be set by users. +- `use_heterogeneous_vocab` currently supports greedy draft sampling only. Probabilistic acceptance (temperature > 0 draft sampling) is not yet supported and will be added in a future release. ## Lossless guarantees of Speculative Decoding diff --git a/docs/features/speculative_decoding/draft_model.md b/docs/features/speculative_decoding/draft_model.md index b4662e6438..636c797324 100644 --- a/docs/features/speculative_decoding/draft_model.md +++ b/docs/features/speculative_decoding/draft_model.md @@ -76,6 +76,34 @@ The code used to request as completions as a client remains unchanged: print(completion) ``` +## Draft Model Method with heterogeneous vocabs + + By default, vLLM requires the draft and target models to share the same vocabulary. Setting `use_heterogeneous_vocab: true` enables the **Token-Level Intersection (TLI)** algorithm, which allows draft models from a different model family with a different tokenizer. + + Currently,`use_heterogeneous_vocab` currently requires `draft_sample_method='greedy'` (the default). Probabilistic draft sampling is not yet supported and will be added in a + future release. + + ```python + from vllm import LLM, SamplingParams + + llm = LLM( + model="Qwen/Qwen3-8B", + speculative_config={ + "method": "draft_model", + "model": "HuggingFaceTB/SmolLM2-135M-Instruct", + "num_speculative_tokens": 3, + "use_heterogeneous_vocab": True, + }, + gpu_memory_utilization=0.5, + ) +outputs = llm.generate(prompts,sampling_params) + +for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") +``` + !!! warning Note: Please use `--speculative-config` to set all configurations related to speculative decoding. The previous method of specifying the model diff --git a/examples/features/speculative_decoding/spec_decode_offline.py b/examples/features/speculative_decoding/spec_decode_offline.py index e60226ba67..593fdb2ad0 100644 --- a/examples/features/speculative_decoding/spec_decode_offline.py +++ b/examples/features/speculative_decoding/spec_decode_offline.py @@ -72,6 +72,7 @@ def parse_args(): parser.add_argument("--max-num-seqs", type=int, default=None) parser.add_argument("--parallel-drafting", action="store_true") parser.add_argument("--allowed-local-media-path", type=str, default="") + parser.add_argument("--use-heterogeneous-vocab", action="store_true") return parser.parse_args() @@ -135,6 +136,7 @@ def main(args): "enforce_eager": args.enforce_eager, "max_model_len": args.max_model_len, "parallel_drafting": args.parallel_drafting, + "use_heterogeneous_vocab": args.use_heterogeneous_vocab, } elif args.method == "mtp": speculative_config = { diff --git a/tests/v1/spec_decode/test_vocab_mapping.py b/tests/v1/spec_decode/test_vocab_mapping.py new file mode 100644 index 0000000000..b8773c5a55 --- /dev/null +++ b/tests/v1/spec_decode/test_vocab_mapping.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +from transformers import AutoTokenizer + +from vllm.v1.spec_decode.vocab_mapping import _detect_space_prefix + + +@pytest.mark.parametrize( + "model_name,expected_prefix", + [ + # BPE tokenizer (GPT-2 family) uses Ġ (U+0120) + ("HuggingFaceTB/SmolLM2-135M-Instruct", ("Ġ",)), + # SentencePiece tokenizer (LLaMA family) uses ▁ (U+2581) + ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", ("▁",)), + # BPE tokenizer (Qwen family) uses Ġ (U+0120) + ("Qwen/Qwen2.5-0.5B-Instruct", ("Ġ",)), + ], +) +def test_detect_space_prefix_real_tokenizers(model_name, expected_prefix): + tokenizer = AutoTokenizer.from_pretrained(model_name) + result = _detect_space_prefix(tokenizer) + assert result == expected_prefix, ( + f"{model_name}: expected {expected_prefix!r}, got {result!r}" + ) + + +def test_detect_space_prefix_fallback_on_failure(): + """When tokenizer lacks encode(), fall back to both known prefixes.""" + + class BrokenTokenizer: + def encode(self, text, **kwargs): + raise RuntimeError("broken") + + result = _detect_space_prefix(BrokenTokenizer()) + assert result == ("Ġ", "▁") + + +def test_detect_space_prefix_empty_encode(): + """When encode returns empty list, fall back.""" + + class EmptyTokenizer: + def encode(self, text, **kwargs): + return [] + + result = _detect_space_prefix(EmptyTokenizer()) + assert result == ("Ġ", "▁") diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index cf7f299eef..983759bc9a 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -139,6 +139,12 @@ class SpeculativeConfig: O(2 * tp_size) per token. Only applies to greedy draft selection in non-tree speculation.""" + use_heterogeneous_vocab: bool = False + """Allow draft and target models to use different vocabularies. + When enabled, builds a token-level intersection at init and constrains + draft logits to shared tokens only (TLI algorithm). Requires + method='draft_model'.""" + # Ngram proposer configuration prompt_lookup_max: int | None = Field(default=None, ge=1) """Maximum size of ngram token window when using Ngram proposer, required @@ -734,7 +740,11 @@ class SpeculativeConfig: self.draft_model_config = ModelConfig( model=self.model, runner="draft", - tokenizer=self.target_model_config.tokenizer, + tokenizer=( + self.model + if self.use_heterogeneous_vocab + else self.target_model_config.tokenizer + ), tokenizer_mode=self.target_model_config.tokenizer_mode, trust_remote_code=self.target_model_config.trust_remote_code, allowed_local_media_path=self.target_model_config.allowed_local_media_path, @@ -1103,7 +1113,20 @@ class SpeculativeConfig: self.draft_parallel_config ) - self.verify_equal_vocab_size_if_draft_model() + if self.use_heterogeneous_vocab and not self.uses_draft_model(): + raise ValueError( + "use_heterogeneous_vocab only works with method='draft_model'" + ) + + if self.use_heterogeneous_vocab and self.draft_sample_method != "greedy": + raise ValueError( + "use_heterogeneous_vocab currently only supports greedy draft " + "sampling. Set draft_sample_method='greedy' (the default) or " + "omit it." + ) + + if not self.use_heterogeneous_vocab: + self.verify_equal_vocab_size_if_draft_model() return self def verify_equal_vocab_size_if_draft_model(self): diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py index a8c8ab03b6..08542f03a1 100644 --- a/vllm/v1/spec_decode/draft_model.py +++ b/vllm/v1/spec_decode/draft_model.py @@ -9,7 +9,9 @@ from vllm.config import VllmConfig from vllm.config.utils import replace from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model +from vllm.tokenizers.registry import get_tokenizer from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer +from vllm.v1.spec_decode.vocab_mapping import VocabMapping logger = init_logger(__name__) @@ -27,9 +29,34 @@ class DraftModelProposer(SpecDecodeBaseProposer): pass_hidden_states_to_model=False, runner=runner, ) - self._raise_if_vocab_size_mismatch() self._raise_if_draft_tp_mismatch() + self.use_heterogeneous_vocab = self.speculative_config.use_heterogeneous_vocab + + spec = self.speculative_config + if self.use_heterogeneous_vocab: + # Heterogeneous vocabularies: build a VocabMapping to translate + # token IDs between the two tokenizers and constrain draft logits + # to the intersection so rejection sampling stays lossless. + target_tokenizer = get_tokenizer( + spec.target_model_config.tokenizer, + trust_remote_code=spec.target_model_config.trust_remote_code, + ) + draft_tokenizer = get_tokenizer( + spec.draft_model_config.model, + trust_remote_code=spec.draft_model_config.trust_remote_code, + ) + self.vocab_mapping: VocabMapping | None = VocabMapping( + target_tokenizer=target_tokenizer, + draft_tokenizer=draft_tokenizer, + target_vocab_size=spec.target_model_config.get_vocab_size(), + draft_vocab_size=spec.draft_model_config.get_vocab_size(), + device=device, + ) + else: + self._raise_if_vocab_size_mismatch() + self.vocab_mapping = None + def _raise_if_vocab_size_mismatch(self): self.speculative_config.verify_equal_vocab_size_if_draft_model() diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index f5e4c27b4f..e43af4aee2 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import dataclasses from importlib.util import find_spec -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import numpy as np import torch @@ -15,6 +15,10 @@ from vllm.config import ( get_layers_from_vllm_config, replace, ) + +if TYPE_CHECKING: + from vllm.v1.spec_decode.vocab_mapping import VocabMapping + from vllm.distributed.eplb.eplb_state import EplbState from vllm.distributed.parallel_state import get_pp_group from vllm.forward_context import set_forward_context @@ -125,6 +129,11 @@ class SpecDecodeBaseProposer: ) self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel + self.use_heterogeneous_vocab: bool = ( + self.speculative_config.use_heterogeneous_vocab + ) + self.vocab_mapping: VocabMapping | None = None + self.max_batch_size = vllm_config.scheduler_config.max_num_seqs self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32) @@ -419,6 +428,12 @@ class SpecDecodeBaseProposer: """Greedy-sample draft tokens from hidden states.""" if self.use_local_argmax_reduction: return self.model.get_top_tokens(hidden_states) + if self.use_heterogeneous_vocab: + logits = self.model.compute_logits(hidden_states) + assert self.vocab_mapping is not None + logits = self.vocab_mapping.constrain_draft_logits(logits) + draft_token_ids = logits.argmax(dim=-1) + return self.vocab_mapping.map_draft_to_target_ids(draft_token_ids) return self.model.compute_logits(hidden_states).argmax(dim=-1) def _sample_from_logits( @@ -457,7 +472,28 @@ class SpecDecodeBaseProposer: if not self._enable_probabilistic_draft_probs or sampling_metadata.all_greedy: return self._greedy_sample(hidden_states), None logits = self.model.compute_logits(hidden_states) - return self._sample_from_logits(logits, sampling_metadata) + if self.use_heterogeneous_vocab: + assert self.vocab_mapping is not None + logits = self.vocab_mapping.constrain_draft_logits(logits) + draft_token_ids, draft_probs = self._sample_from_logits( + logits, sampling_metadata + ) + if self.use_heterogeneous_vocab: + assert self.vocab_mapping is not None + draft_token_ids = self.vocab_mapping.map_draft_to_target_ids( + draft_token_ids + ) + # Config validation ensures draft_sample_method == "greedy" when + # use_heterogeneous_vocab is True, so this branch should never be + # reached. Kept as a safety fallback until probabilistic rejection + # sampling with heterogeneous vocabularies is implemented. + # TODO: remap draft_probs to target-vocab space for lossless + # probabilistic rejection sampling with heterogeneous vocabularies. + assert draft_probs is None, ( + "probabilistic draft sampling is not supported with " + "use_heterogeneous_vocab" + ) + return draft_token_ids, draft_probs def take_last_draft_probs(self) -> torch.Tensor | None: return self._last_draft_probs @@ -647,6 +683,11 @@ class SpecDecodeBaseProposer: # tensor.argmax() returns int64 by default. input_ids = draft_token_ids_list[-1].int() + if self.use_heterogeneous_vocab: + # Map target token IDs to draft vocab space (TLI algorithm) + assert self.vocab_mapping is not None + input_ids = self.vocab_mapping.map_target_to_draft_ids(input_ids) + if not self.constant_draft_positions: positions = self._update_positions_dependent_metadata( positions, @@ -785,6 +826,13 @@ class SpecDecodeBaseProposer: cad: CommonAttentionMetadata, num_rejected_tokens_gpu: torch.Tensor | None, ) -> tuple[int, torch.Tensor, CommonAttentionMetadata]: + # Map target token IDs to draft vocab space (TLI algorithm) + if self.use_heterogeneous_vocab: + assert self.vocab_mapping is not None + target_token_ids = self.vocab_mapping.map_target_to_draft_ids( + target_token_ids + ) + next_token_ids = self.vocab_mapping.map_target_to_draft_ids(next_token_ids) if not self.needs_extra_input_slots: # Default EAGLE pathway: no reshaping of input tensors needed. # Simply rotate the input ids and leave the positions unchanged, diff --git a/vllm/v1/spec_decode/vocab_mapping.py b/vllm/v1/spec_decode/vocab_mapping.py new file mode 100644 index 0000000000..9a6bbe7cfa --- /dev/null +++ b/vllm/v1/spec_decode/vocab_mapping.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def _detect_space_prefix(tokenizer) -> tuple[str, ...]: + """Detect the space-prefix character(s) by tokenizing a literal space. + + Different tokenizer families mark word-initial spaces differently: + BPE uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581). Probing at + runtime avoids hardcoding assumptions and correctly handles mixed-family + pairs (e.g. BPE draft + SentencePiece target). + """ + try: + space_ids = tokenizer.encode(" a", add_special_tokens=False) + if space_ids: + tok_str = tokenizer.convert_ids_to_tokens(space_ids[0]) + if ( + isinstance(tok_str, str) + and len(tok_str) > 1 + and tok_str.endswith("a") + and tok_str[0] not in (" ", " ") + ): + return (tok_str[:-1],) + except Exception: + pass + # Fallback: cover both BPE (Ġ U+0120) and SentencePiece (▁ U+2581) + return ("\u0120", "\u2581") + + +def _normalize_token(token: str, space_prefixes: tuple[str, ...]) -> str: + for prefix in space_prefixes: + if token.startswith(prefix): + return " " + token[len(prefix) :] + return token + + +def _get_unk_token_id(tokenizer, role: str) -> int: + """Return a safe fallback token ID for out-of-intersection tokens. + + Preferred: unk_token_id → eos_token_id → ValueError. + Checking with ``is not None`` is required because token ID 0 is a valid + (and common) unk ID on many tokenizers; using ``or 0`` would silently + mishandle those cases. + """ + unk = getattr(tokenizer, "unk_token_id", None) + if unk is not None: + return unk + eos = getattr(tokenizer, "eos_token_id", None) + if eos is not None: + logger.warning( + "VocabMapping: %s has no unk_token_id; " + "falling back to eos_token_id=%d for out-of-intersection tokens", + role, + eos, + ) + return eos + raise ValueError( + f"VocabMapping: {role} has neither unk_token_id nor eos_token_id; " + "cannot safely map out-of-intersection tokens" + ) + + +class VocabMapping: + def __init__( + self, + target_tokenizer, + draft_tokenizer, + target_vocab_size, + draft_vocab_size, + device, + ): + self.target_vocab_size = target_vocab_size + self.draft_vocab_size = draft_vocab_size + self.device = device + self.target_unk_token_id = _get_unk_token_id( + target_tokenizer, "target tokenizer" + ) + self.draft_unk_token_id = _get_unk_token_id(draft_tokenizer, "draft tokenizer") + + target_prefixes = _detect_space_prefix(target_tokenizer) + draft_prefixes = _detect_space_prefix(draft_tokenizer) + + target_vocab = target_tokenizer.get_vocab() + draft_vocab = draft_tokenizer.get_vocab() + + target_normalized = {} + for token, tid in target_vocab.items(): + norm = _normalize_token(token, target_prefixes) + if norm not in target_normalized: + target_normalized[norm] = tid + + draft_normalized = {} + for token, tid in draft_vocab.items(): + norm = _normalize_token(token, draft_prefixes) + if norm not in draft_normalized: + draft_normalized[norm] = tid + + common_tokens = set(target_normalized.keys()) & set(draft_normalized.keys()) + + draft_to_target = torch.full((draft_vocab_size,), -1, dtype=torch.long) + target_to_draft = torch.full((target_vocab_size,), -1, dtype=torch.long) + intersection_mask_draft = torch.zeros(draft_vocab_size, dtype=torch.bool) + + for norm_token in common_tokens: + t_id = target_normalized[norm_token] + d_id = draft_normalized[norm_token] + if t_id < target_vocab_size and d_id < draft_vocab_size: + draft_to_target[d_id] = t_id + target_to_draft[t_id] = d_id + intersection_mask_draft[d_id] = True + + self.draft_to_target_ids = draft_to_target.to(device) + self.target_to_draft_ids = target_to_draft.to(device) + self.intersection_mask_draft = intersection_mask_draft.to(device) + self.intersection_size = int(intersection_mask_draft.sum().item()) + + logger.info( + "VocabMapping initialized: target_vocab=%d, draft_vocab=%d, " + "intersection=%d (%.1f%% of draft, %.1f%% of target)", + target_vocab_size, + draft_vocab_size, + self.intersection_size, + 100.0 * self.intersection_size / max(draft_vocab_size, 1), + 100.0 * self.intersection_size / max(target_vocab_size, 1), + ) + + if self.intersection_size < 100: + logger.warning( + "Very small vocabulary intersection (%d tokens).", + self.intersection_size, + ) + + def map_target_to_draft_ids(self, target_ids): + draft_ids = self.target_to_draft_ids[target_ids] # new tensor; no clone needed + missing = draft_ids == -1 + if missing.any(): + draft_ids[missing] = self.draft_unk_token_id + return draft_ids.to(target_ids.dtype) + + def map_draft_to_target_ids(self, draft_ids): + target_ids = self.draft_to_target_ids[draft_ids] # new tensor; no clone needed + missing = target_ids == -1 + if missing.any(): + target_ids[missing] = self.target_unk_token_id + return target_ids.to(draft_ids.dtype) + + def constrain_draft_logits(self, logits): + # masked_fill returns a new tensor; no clone needed + return logits.masked_fill(~self.intersection_mask_draft, float("-inf"))