[Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174)
Signed-off-by: wan-danfeng <wandanfeng0802@gmail.com> Signed-off-by: Wonderful <wandanfeng0802@gmail.com> Co-authored-by: Wan_DF <wonderful199082@126.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Wan_DF
Benjamin Chislett
Benjamin Chislett
parent
8357226f4f
commit
3af8789559
@@ -86,6 +86,7 @@ only apply to model-based methods such as `draft_model`, `mtp`, `eagle3`, and
|
||||
| `parallel_drafting` | `boolean` | `false` | Enable parallel draft token generation. Only compatible with EAGLE and draft-model methods. |
|
||||
| `rejection_sample_method` | `string` | `strict` | `strict`, `probabilistic`, or `synthetic`. |
|
||||
| `synthetic_acceptance_rate` | `float` | `None` | Average acceptance rate to target when `rejection_sample_method` is `synthetic`. Valid range is `[0, 1]`. |
|
||||
| `use_heterogeneous_vocab` | `boolean` | `false` | Allow draft and target models with different vocabularies. Builds a token-level intersection at initialisation and constrains draft logits to shared tokens only. Only compatible with `method=draft_model`. Probabilistic draft sampling (`draft_sample_method='probabilistic'`) is not yet supported when this option is enabled. |
|
||||
|
||||
!!! note
|
||||
Gemma 4 assistant checkpoints are handled as Gemma 4 MTP speculators, not
|
||||
@@ -142,6 +143,33 @@ vllm serve <target-model> \
|
||||
}'
|
||||
```
|
||||
|
||||
#### Cross-Vocabulary Draft Models (TLI)
|
||||
|
||||
By default, vLLM requires the draft and target models to share the same
|
||||
vocabulary. Setting `use_heterogeneous_vocab: true` enables the
|
||||
**Token-Level Intersection (TLI)** algorithm, which allows draft models
|
||||
from a different model family with a different tokenizer.
|
||||
|
||||
At initialisation, vLLM builds a mapping between the two vocabularies by
|
||||
normalising token strings and computing their intersection. Draft logits are
|
||||
constrained to the shared tokens before sampling, and the sampled token IDs
|
||||
are translated to the target vocabulary before rejection sampling.
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-8B",
|
||||
speculative_config={
|
||||
"method": "draft_model",
|
||||
"model": "HuggingFaceTB/SmolLM2-135M-Instruct",
|
||||
"num_speculative_tokens": 3,
|
||||
"use_heterogeneous_vocab": True,
|
||||
},
|
||||
gpu_memory_utilization=0.5,
|
||||
)
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- `--speculative-config` expects a JSON object on the CLI. In YAML config
|
||||
@@ -153,6 +181,7 @@ vllm serve <target-model> \
|
||||
- Internal fields such as `target_model_config`, `draft_model_config`,
|
||||
`target_parallel_config`, `draft_parallel_config`, and `draft_load_config`
|
||||
are populated by vLLM and are not intended to be set by users.
|
||||
- `use_heterogeneous_vocab` currently supports greedy draft sampling only. Probabilistic acceptance (temperature > 0 draft sampling) is not yet supported and will be added in a future release.
|
||||
|
||||
## Lossless guarantees of Speculative Decoding
|
||||
|
||||
|
||||
@@ -76,6 +76,34 @@ The code used to request as completions as a client remains unchanged:
|
||||
print(completion)
|
||||
```
|
||||
|
||||
## Draft Model Method with heterogeneous vocabs
|
||||
|
||||
By default, vLLM requires the draft and target models to share the same vocabulary. Setting `use_heterogeneous_vocab: true` enables the **Token-Level Intersection (TLI)** algorithm, which allows draft models from a different model family with a different tokenizer.
|
||||
|
||||
Currently,`use_heterogeneous_vocab` currently requires `draft_sample_method='greedy'` (the default). Probabilistic draft sampling is not yet supported and will be added in a
|
||||
future release.
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-8B",
|
||||
speculative_config={
|
||||
"method": "draft_model",
|
||||
"model": "HuggingFaceTB/SmolLM2-135M-Instruct",
|
||||
"num_speculative_tokens": 3,
|
||||
"use_heterogeneous_vocab": True,
|
||||
},
|
||||
gpu_memory_utilization=0.5,
|
||||
)
|
||||
outputs = llm.generate(prompts,sampling_params)
|
||||
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
```
|
||||
|
||||
!!! warning
|
||||
Note: Please use `--speculative-config` to set all configurations related
|
||||
to speculative decoding. The previous method of specifying the model
|
||||
|
||||
@@ -72,6 +72,7 @@ def parse_args():
|
||||
parser.add_argument("--max-num-seqs", type=int, default=None)
|
||||
parser.add_argument("--parallel-drafting", action="store_true")
|
||||
parser.add_argument("--allowed-local-media-path", type=str, default="")
|
||||
parser.add_argument("--use-heterogeneous-vocab", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -135,6 +136,7 @@ def main(args):
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"max_model_len": args.max_model_len,
|
||||
"parallel_drafting": args.parallel_drafting,
|
||||
"use_heterogeneous_vocab": args.use_heterogeneous_vocab,
|
||||
}
|
||||
elif args.method == "mtp":
|
||||
speculative_config = {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.v1.spec_decode.vocab_mapping import _detect_space_prefix
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,expected_prefix",
|
||||
[
|
||||
# BPE tokenizer (GPT-2 family) uses Ġ (U+0120)
|
||||
("HuggingFaceTB/SmolLM2-135M-Instruct", ("Ġ",)),
|
||||
# SentencePiece tokenizer (LLaMA family) uses ▁ (U+2581)
|
||||
("TinyLlama/TinyLlama-1.1B-Chat-v1.0", ("▁",)),
|
||||
# BPE tokenizer (Qwen family) uses Ġ (U+0120)
|
||||
("Qwen/Qwen2.5-0.5B-Instruct", ("Ġ",)),
|
||||
],
|
||||
)
|
||||
def test_detect_space_prefix_real_tokenizers(model_name, expected_prefix):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
result = _detect_space_prefix(tokenizer)
|
||||
assert result == expected_prefix, (
|
||||
f"{model_name}: expected {expected_prefix!r}, got {result!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_detect_space_prefix_fallback_on_failure():
|
||||
"""When tokenizer lacks encode(), fall back to both known prefixes."""
|
||||
|
||||
class BrokenTokenizer:
|
||||
def encode(self, text, **kwargs):
|
||||
raise RuntimeError("broken")
|
||||
|
||||
result = _detect_space_prefix(BrokenTokenizer())
|
||||
assert result == ("Ġ", "▁")
|
||||
|
||||
|
||||
def test_detect_space_prefix_empty_encode():
|
||||
"""When encode returns empty list, fall back."""
|
||||
|
||||
class EmptyTokenizer:
|
||||
def encode(self, text, **kwargs):
|
||||
return []
|
||||
|
||||
result = _detect_space_prefix(EmptyTokenizer())
|
||||
assert result == ("Ġ", "▁")
|
||||
@@ -139,6 +139,12 @@ class SpeculativeConfig:
|
||||
O(2 * tp_size) per token. Only applies to greedy draft selection in
|
||||
non-tree speculation."""
|
||||
|
||||
use_heterogeneous_vocab: bool = False
|
||||
"""Allow draft and target models to use different vocabularies.
|
||||
When enabled, builds a token-level intersection at init and constrains
|
||||
draft logits to shared tokens only (TLI algorithm). Requires
|
||||
method='draft_model'."""
|
||||
|
||||
# Ngram proposer configuration
|
||||
prompt_lookup_max: int | None = Field(default=None, ge=1)
|
||||
"""Maximum size of ngram token window when using Ngram proposer, required
|
||||
@@ -734,7 +740,11 @@ class SpeculativeConfig:
|
||||
self.draft_model_config = ModelConfig(
|
||||
model=self.model,
|
||||
runner="draft",
|
||||
tokenizer=self.target_model_config.tokenizer,
|
||||
tokenizer=(
|
||||
self.model
|
||||
if self.use_heterogeneous_vocab
|
||||
else self.target_model_config.tokenizer
|
||||
),
|
||||
tokenizer_mode=self.target_model_config.tokenizer_mode,
|
||||
trust_remote_code=self.target_model_config.trust_remote_code,
|
||||
allowed_local_media_path=self.target_model_config.allowed_local_media_path,
|
||||
@@ -1103,7 +1113,20 @@ class SpeculativeConfig:
|
||||
self.draft_parallel_config
|
||||
)
|
||||
|
||||
self.verify_equal_vocab_size_if_draft_model()
|
||||
if self.use_heterogeneous_vocab and not self.uses_draft_model():
|
||||
raise ValueError(
|
||||
"use_heterogeneous_vocab only works with method='draft_model'"
|
||||
)
|
||||
|
||||
if self.use_heterogeneous_vocab and self.draft_sample_method != "greedy":
|
||||
raise ValueError(
|
||||
"use_heterogeneous_vocab currently only supports greedy draft "
|
||||
"sampling. Set draft_sample_method='greedy' (the default) or "
|
||||
"omit it."
|
||||
)
|
||||
|
||||
if not self.use_heterogeneous_vocab:
|
||||
self.verify_equal_vocab_size_if_draft_model()
|
||||
return self
|
||||
|
||||
def verify_equal_vocab_size_if_draft_model(self):
|
||||
|
||||
@@ -9,7 +9,9 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.utils import replace
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.tokenizers.registry import get_tokenizer
|
||||
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
|
||||
from vllm.v1.spec_decode.vocab_mapping import VocabMapping
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -27,9 +29,34 @@ class DraftModelProposer(SpecDecodeBaseProposer):
|
||||
pass_hidden_states_to_model=False,
|
||||
runner=runner,
|
||||
)
|
||||
self._raise_if_vocab_size_mismatch()
|
||||
self._raise_if_draft_tp_mismatch()
|
||||
|
||||
self.use_heterogeneous_vocab = self.speculative_config.use_heterogeneous_vocab
|
||||
|
||||
spec = self.speculative_config
|
||||
if self.use_heterogeneous_vocab:
|
||||
# Heterogeneous vocabularies: build a VocabMapping to translate
|
||||
# token IDs between the two tokenizers and constrain draft logits
|
||||
# to the intersection so rejection sampling stays lossless.
|
||||
target_tokenizer = get_tokenizer(
|
||||
spec.target_model_config.tokenizer,
|
||||
trust_remote_code=spec.target_model_config.trust_remote_code,
|
||||
)
|
||||
draft_tokenizer = get_tokenizer(
|
||||
spec.draft_model_config.model,
|
||||
trust_remote_code=spec.draft_model_config.trust_remote_code,
|
||||
)
|
||||
self.vocab_mapping: VocabMapping | None = VocabMapping(
|
||||
target_tokenizer=target_tokenizer,
|
||||
draft_tokenizer=draft_tokenizer,
|
||||
target_vocab_size=spec.target_model_config.get_vocab_size(),
|
||||
draft_vocab_size=spec.draft_model_config.get_vocab_size(),
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self._raise_if_vocab_size_mismatch()
|
||||
self.vocab_mapping = None
|
||||
|
||||
def _raise_if_vocab_size_mismatch(self):
|
||||
self.speculative_config.verify_equal_vocab_size_if_draft_model()
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from importlib.util import find_spec
|
||||
from typing import Any, cast
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -15,6 +15,10 @@ from vllm.config import (
|
||||
get_layers_from_vllm_config,
|
||||
replace,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.spec_decode.vocab_mapping import VocabMapping
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.forward_context import set_forward_context
|
||||
@@ -125,6 +129,11 @@ class SpecDecodeBaseProposer:
|
||||
)
|
||||
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
|
||||
|
||||
self.use_heterogeneous_vocab: bool = (
|
||||
self.speculative_config.use_heterogeneous_vocab
|
||||
)
|
||||
self.vocab_mapping: VocabMapping | None = None
|
||||
|
||||
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
|
||||
@@ -419,6 +428,12 @@ class SpecDecodeBaseProposer:
|
||||
"""Greedy-sample draft tokens from hidden states."""
|
||||
if self.use_local_argmax_reduction:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
if self.use_heterogeneous_vocab:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
assert self.vocab_mapping is not None
|
||||
logits = self.vocab_mapping.constrain_draft_logits(logits)
|
||||
draft_token_ids = logits.argmax(dim=-1)
|
||||
return self.vocab_mapping.map_draft_to_target_ids(draft_token_ids)
|
||||
return self.model.compute_logits(hidden_states).argmax(dim=-1)
|
||||
|
||||
def _sample_from_logits(
|
||||
@@ -457,7 +472,28 @@ class SpecDecodeBaseProposer:
|
||||
if not self._enable_probabilistic_draft_probs or sampling_metadata.all_greedy:
|
||||
return self._greedy_sample(hidden_states), None
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
return self._sample_from_logits(logits, sampling_metadata)
|
||||
if self.use_heterogeneous_vocab:
|
||||
assert self.vocab_mapping is not None
|
||||
logits = self.vocab_mapping.constrain_draft_logits(logits)
|
||||
draft_token_ids, draft_probs = self._sample_from_logits(
|
||||
logits, sampling_metadata
|
||||
)
|
||||
if self.use_heterogeneous_vocab:
|
||||
assert self.vocab_mapping is not None
|
||||
draft_token_ids = self.vocab_mapping.map_draft_to_target_ids(
|
||||
draft_token_ids
|
||||
)
|
||||
# Config validation ensures draft_sample_method == "greedy" when
|
||||
# use_heterogeneous_vocab is True, so this branch should never be
|
||||
# reached. Kept as a safety fallback until probabilistic rejection
|
||||
# sampling with heterogeneous vocabularies is implemented.
|
||||
# TODO: remap draft_probs to target-vocab space for lossless
|
||||
# probabilistic rejection sampling with heterogeneous vocabularies.
|
||||
assert draft_probs is None, (
|
||||
"probabilistic draft sampling is not supported with "
|
||||
"use_heterogeneous_vocab"
|
||||
)
|
||||
return draft_token_ids, draft_probs
|
||||
|
||||
def take_last_draft_probs(self) -> torch.Tensor | None:
|
||||
return self._last_draft_probs
|
||||
@@ -647,6 +683,11 @@ class SpecDecodeBaseProposer:
|
||||
# tensor.argmax() returns int64 by default.
|
||||
input_ids = draft_token_ids_list[-1].int()
|
||||
|
||||
if self.use_heterogeneous_vocab:
|
||||
# Map target token IDs to draft vocab space (TLI algorithm)
|
||||
assert self.vocab_mapping is not None
|
||||
input_ids = self.vocab_mapping.map_target_to_draft_ids(input_ids)
|
||||
|
||||
if not self.constant_draft_positions:
|
||||
positions = self._update_positions_dependent_metadata(
|
||||
positions,
|
||||
@@ -785,6 +826,13 @@ class SpecDecodeBaseProposer:
|
||||
cad: CommonAttentionMetadata,
|
||||
num_rejected_tokens_gpu: torch.Tensor | None,
|
||||
) -> tuple[int, torch.Tensor, CommonAttentionMetadata]:
|
||||
# Map target token IDs to draft vocab space (TLI algorithm)
|
||||
if self.use_heterogeneous_vocab:
|
||||
assert self.vocab_mapping is not None
|
||||
target_token_ids = self.vocab_mapping.map_target_to_draft_ids(
|
||||
target_token_ids
|
||||
)
|
||||
next_token_ids = self.vocab_mapping.map_target_to_draft_ids(next_token_ids)
|
||||
if not self.needs_extra_input_slots:
|
||||
# Default EAGLE pathway: no reshaping of input tensors needed.
|
||||
# Simply rotate the input ids and leave the positions unchanged,
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _detect_space_prefix(tokenizer) -> tuple[str, ...]:
|
||||
"""Detect the space-prefix character(s) by tokenizing a literal space.
|
||||
|
||||
Different tokenizer families mark word-initial spaces differently:
|
||||
BPE uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581). Probing at
|
||||
runtime avoids hardcoding assumptions and correctly handles mixed-family
|
||||
pairs (e.g. BPE draft + SentencePiece target).
|
||||
"""
|
||||
try:
|
||||
space_ids = tokenizer.encode(" a", add_special_tokens=False)
|
||||
if space_ids:
|
||||
tok_str = tokenizer.convert_ids_to_tokens(space_ids[0])
|
||||
if (
|
||||
isinstance(tok_str, str)
|
||||
and len(tok_str) > 1
|
||||
and tok_str.endswith("a")
|
||||
and tok_str[0] not in (" ", " ")
|
||||
):
|
||||
return (tok_str[:-1],)
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback: cover both BPE (Ġ U+0120) and SentencePiece (▁ U+2581)
|
||||
return ("\u0120", "\u2581")
|
||||
|
||||
|
||||
def _normalize_token(token: str, space_prefixes: tuple[str, ...]) -> str:
|
||||
for prefix in space_prefixes:
|
||||
if token.startswith(prefix):
|
||||
return " " + token[len(prefix) :]
|
||||
return token
|
||||
|
||||
|
||||
def _get_unk_token_id(tokenizer, role: str) -> int:
|
||||
"""Return a safe fallback token ID for out-of-intersection tokens.
|
||||
|
||||
Preferred: unk_token_id → eos_token_id → ValueError.
|
||||
Checking with ``is not None`` is required because token ID 0 is a valid
|
||||
(and common) unk ID on many tokenizers; using ``or 0`` would silently
|
||||
mishandle those cases.
|
||||
"""
|
||||
unk = getattr(tokenizer, "unk_token_id", None)
|
||||
if unk is not None:
|
||||
return unk
|
||||
eos = getattr(tokenizer, "eos_token_id", None)
|
||||
if eos is not None:
|
||||
logger.warning(
|
||||
"VocabMapping: %s has no unk_token_id; "
|
||||
"falling back to eos_token_id=%d for out-of-intersection tokens",
|
||||
role,
|
||||
eos,
|
||||
)
|
||||
return eos
|
||||
raise ValueError(
|
||||
f"VocabMapping: {role} has neither unk_token_id nor eos_token_id; "
|
||||
"cannot safely map out-of-intersection tokens"
|
||||
)
|
||||
|
||||
|
||||
class VocabMapping:
|
||||
def __init__(
|
||||
self,
|
||||
target_tokenizer,
|
||||
draft_tokenizer,
|
||||
target_vocab_size,
|
||||
draft_vocab_size,
|
||||
device,
|
||||
):
|
||||
self.target_vocab_size = target_vocab_size
|
||||
self.draft_vocab_size = draft_vocab_size
|
||||
self.device = device
|
||||
self.target_unk_token_id = _get_unk_token_id(
|
||||
target_tokenizer, "target tokenizer"
|
||||
)
|
||||
self.draft_unk_token_id = _get_unk_token_id(draft_tokenizer, "draft tokenizer")
|
||||
|
||||
target_prefixes = _detect_space_prefix(target_tokenizer)
|
||||
draft_prefixes = _detect_space_prefix(draft_tokenizer)
|
||||
|
||||
target_vocab = target_tokenizer.get_vocab()
|
||||
draft_vocab = draft_tokenizer.get_vocab()
|
||||
|
||||
target_normalized = {}
|
||||
for token, tid in target_vocab.items():
|
||||
norm = _normalize_token(token, target_prefixes)
|
||||
if norm not in target_normalized:
|
||||
target_normalized[norm] = tid
|
||||
|
||||
draft_normalized = {}
|
||||
for token, tid in draft_vocab.items():
|
||||
norm = _normalize_token(token, draft_prefixes)
|
||||
if norm not in draft_normalized:
|
||||
draft_normalized[norm] = tid
|
||||
|
||||
common_tokens = set(target_normalized.keys()) & set(draft_normalized.keys())
|
||||
|
||||
draft_to_target = torch.full((draft_vocab_size,), -1, dtype=torch.long)
|
||||
target_to_draft = torch.full((target_vocab_size,), -1, dtype=torch.long)
|
||||
intersection_mask_draft = torch.zeros(draft_vocab_size, dtype=torch.bool)
|
||||
|
||||
for norm_token in common_tokens:
|
||||
t_id = target_normalized[norm_token]
|
||||
d_id = draft_normalized[norm_token]
|
||||
if t_id < target_vocab_size and d_id < draft_vocab_size:
|
||||
draft_to_target[d_id] = t_id
|
||||
target_to_draft[t_id] = d_id
|
||||
intersection_mask_draft[d_id] = True
|
||||
|
||||
self.draft_to_target_ids = draft_to_target.to(device)
|
||||
self.target_to_draft_ids = target_to_draft.to(device)
|
||||
self.intersection_mask_draft = intersection_mask_draft.to(device)
|
||||
self.intersection_size = int(intersection_mask_draft.sum().item())
|
||||
|
||||
logger.info(
|
||||
"VocabMapping initialized: target_vocab=%d, draft_vocab=%d, "
|
||||
"intersection=%d (%.1f%% of draft, %.1f%% of target)",
|
||||
target_vocab_size,
|
||||
draft_vocab_size,
|
||||
self.intersection_size,
|
||||
100.0 * self.intersection_size / max(draft_vocab_size, 1),
|
||||
100.0 * self.intersection_size / max(target_vocab_size, 1),
|
||||
)
|
||||
|
||||
if self.intersection_size < 100:
|
||||
logger.warning(
|
||||
"Very small vocabulary intersection (%d tokens).",
|
||||
self.intersection_size,
|
||||
)
|
||||
|
||||
def map_target_to_draft_ids(self, target_ids):
|
||||
draft_ids = self.target_to_draft_ids[target_ids] # new tensor; no clone needed
|
||||
missing = draft_ids == -1
|
||||
if missing.any():
|
||||
draft_ids[missing] = self.draft_unk_token_id
|
||||
return draft_ids.to(target_ids.dtype)
|
||||
|
||||
def map_draft_to_target_ids(self, draft_ids):
|
||||
target_ids = self.draft_to_target_ids[draft_ids] # new tensor; no clone needed
|
||||
missing = target_ids == -1
|
||||
if missing.any():
|
||||
target_ids[missing] = self.target_unk_token_id
|
||||
return target_ids.to(draft_ids.dtype)
|
||||
|
||||
def constrain_draft_logits(self, logits):
|
||||
# masked_fill returns a new tensor; no clone needed
|
||||
return logits.masked_fill(~self.intersection_mask_draft, float("-inf"))
|
||||
Reference in New Issue
Block a user