diff --git a/docs/features/speculative_decoding/speculators.md b/docs/features/speculative_decoding/speculators.md index 864efd46ae..cdf6a5f843 100644 --- a/docs/features/speculative_decoding/speculators.md +++ b/docs/features/speculative_decoding/speculators.md @@ -12,6 +12,17 @@ Speculators provides the following key features: - **Standardized, extensible format**: Provides a Hugging Face-compatible format for defining speculative models, with tools to convert from external research repositories into a standard speculators format for easy adoption. - **Seamless vLLM Integration**: Built for direct deployment into vLLM, enabling low-latency, production-grade inference with minimal overhead. +## Verifier model references + +When serving a Speculators checkpoint directly, `speculators_config.verifier.name_or_path` must be a remote model repository ID. References to local files or directories, including local paths that look like repository IDs, are rejected. + +To use a local verifier, select it explicitly and configure the Speculators checkpoint as the draft model: + +```bash +vllm serve /path/to/verifier --speculative-config \ + '{"model": "/path/to/speculator", "method": "eagle3", "num_speculative_tokens": 1}' +``` + ## Why use Speculators? Large language models generate text one token at a time, which creates a fundamental bottleneck: each token requires a full forward pass through the model, leaving GPU compute underutilized while waiting for memory-bound operations. diff --git a/tests/transformers_utils/test_speculators_override.py b/tests/transformers_utils/test_speculators_override.py index 07f607cb88..1495d5a87c 100644 --- a/tests/transformers_utils/test_speculators_override.py +++ b/tests/transformers_utils/test_speculators_override.py @@ -10,11 +10,17 @@ runtime fields from the user are preserved while ``model`` stays auto-detected from the speculators format. """ +import json +from copy import deepcopy +from typing import Any from unittest.mock import patch +import pytest + +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs from vllm.transformers_utils.config import maybe_override_with_speculators -_SPECULATORS_CONFIG_DICT = { +_SPECULATORS_CONFIG_DICT: dict[str, Any] = { "architectures": ["DFlashDraftModel"], "speculators_config": { "algorithm": "dflash", @@ -154,3 +160,78 @@ def test_non_speculators_model_passes_user_config_through_unchanged(): assert model == "fake-org/plain-llama" assert tok is None assert spec == user_overrides + + +@pytest.mark.parametrize( + "verifier", + [ + "/abs/path/to/other_tenant_private", + "../private-model", + "./private-model", + "file:///private-model", + "C:\\private-model", + "", + None, + [], + {}, + 1, + ], +) +def test_speculators_rejects_invalid_verifier_references(verifier): + config = deepcopy(_SPECULATORS_CONFIG_DICT) + config["speculators_config"]["verifier"]["name_or_path"] = verifier + with ( + _patched_get_config_dict(config), + pytest.raises(ValueError, match="Speculators verifier must"), + ): + maybe_override_with_speculators("fake-org/draft", None, False) + + +@pytest.mark.parametrize("verifier", ["private-model", "fake-org/fake-verifier"]) +@pytest.mark.parametrize("symlink", [False, True]) +def test_speculators_rejects_local_verifiers_with_valid_repo_names( + tmp_path, monkeypatch, verifier, symlink +): + monkeypatch.chdir(tmp_path) + local_verifier = tmp_path / verifier + local_verifier.parent.mkdir(parents=True, exist_ok=True) + if symlink: + local_verifier.symlink_to(tmp_path / "private-target", target_is_directory=True) + else: + local_verifier.mkdir() + + config = deepcopy(_SPECULATORS_CONFIG_DICT) + config["speculators_config"]["verifier"]["name_or_path"] = verifier + with ( + _patched_get_config_dict(config), + pytest.raises(ValueError, match="Speculators verifier must"), + ): + maybe_override_with_speculators("fake-org/draft", None, False) + + +def test_non_speculators_local_model_remains_operator_selected(tmp_path): + plain_config = {"architectures": ["LlamaForCausalLM"], "hidden_size": 64} + with _patched_get_config_dict(plain_config): + model, tokenizer, _ = maybe_override_with_speculators( + str(tmp_path), + str(tmp_path), + False, + vllm_speculative_config={"model": "fake-org/draft", "method": "eagle3"}, + ) + assert model == tokenizer == str(tmp_path) + + +@pytest.mark.parametrize("engine_args_cls", [EngineArgs, AsyncEngineArgs]) +def test_speculators_rejects_local_verifier_before_model_loading( + tmp_path, engine_args_cls +): + private_model = tmp_path / "private-model" + private_model.mkdir() + config = deepcopy(_SPECULATORS_CONFIG_DICT) + config["speculators_config"]["verifier"]["name_or_path"] = str(private_model) + draft = tmp_path / "draft" + draft.mkdir() + (draft / "config.json").write_text(json.dumps(config), encoding="utf-8") + + with pytest.raises(ValueError, match="Speculators verifier must"): + engine_args_cls(model=str(draft)).create_engine_config() diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index c2152392ea..5b6586269c 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -14,6 +14,7 @@ import huggingface_hub import torch import transformers.configuration_utils as hf_configuration_utils from huggingface_hub import constants +from huggingface_hub.utils import HFValidationError, validate_repo_id from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE from transformers import GenerationConfig, PreTrainedConfig from transformers.configuration_utils import ALLOWED_LAYER_TYPES @@ -844,6 +845,19 @@ def maybe_override_with_speculators( # Override model and tokenizer with the verifier model from config verifier_model = speculators_config["verifier"]["name_or_path"] + verifier_error = ( + "Speculators verifier must name a remote model repository. " + "Select local verifiers explicitly with --model and --speculative-config." + ) + if not isinstance(verifier_model, str): + raise ValueError(verifier_error) + try: + validate_repo_id(verifier_model) + except HFValidationError as exc: + raise ValueError(verifier_error) from exc + verifier_path = Path(verifier_model) + if verifier_path.exists() or verifier_path.is_symlink(): + raise ValueError(verifier_error) model = tokenizer = verifier_model return model, tokenizer, speculative_config