[Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174)

Signed-off-by: wan-danfeng <wandanfeng0802@gmail.com>
Signed-off-by: Wonderful <wandanfeng0802@gmail.com>
Co-authored-by: Wan_DF <wonderful199082@126.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Wonderful
2026-07-02 01:34:20 -07:00
committed by GitHub
co-authored by Wan_DF Benjamin Chislett Benjamin Chislett
parent 8357226f4f
commit 3af8789559
8 changed files with 364 additions and 5 deletions
+28 -1
View File
@@ -9,7 +9,9 @@ from vllm.config import VllmConfig
from vllm.config.utils import replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.tokenizers.registry import get_tokenizer
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.spec_decode.vocab_mapping import VocabMapping
logger = init_logger(__name__)
@@ -27,9 +29,34 @@ class DraftModelProposer(SpecDecodeBaseProposer):
pass_hidden_states_to_model=False,
runner=runner,
)
self._raise_if_vocab_size_mismatch()
self._raise_if_draft_tp_mismatch()
self.use_heterogeneous_vocab = self.speculative_config.use_heterogeneous_vocab
spec = self.speculative_config
if self.use_heterogeneous_vocab:
# Heterogeneous vocabularies: build a VocabMapping to translate
# token IDs between the two tokenizers and constrain draft logits
# to the intersection so rejection sampling stays lossless.
target_tokenizer = get_tokenizer(
spec.target_model_config.tokenizer,
trust_remote_code=spec.target_model_config.trust_remote_code,
)
draft_tokenizer = get_tokenizer(
spec.draft_model_config.model,
trust_remote_code=spec.draft_model_config.trust_remote_code,
)
self.vocab_mapping: VocabMapping | None = VocabMapping(
target_tokenizer=target_tokenizer,
draft_tokenizer=draft_tokenizer,
target_vocab_size=spec.target_model_config.get_vocab_size(),
draft_vocab_size=spec.draft_model_config.get_vocab_size(),
device=device,
)
else:
self._raise_if_vocab_size_mismatch()
self.vocab_mapping = None
def _raise_if_vocab_size_mismatch(self):
self.speculative_config.verify_equal_vocab_size_if_draft_model()