[Security] Reject local verifier references in Speculators metadata (#61587)
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
bf6c80c7e8
commit
5d88c034f6
@@ -12,6 +12,17 @@ Speculators provides the following key features:
|
||||
- **Standardized, extensible format**: Provides a Hugging Face-compatible format for defining speculative models, with tools to convert from external research repositories into a standard speculators format for easy adoption.
|
||||
- **Seamless vLLM Integration**: Built for direct deployment into vLLM, enabling low-latency, production-grade inference with minimal overhead.
|
||||
|
||||
## Verifier model references
|
||||
|
||||
When serving a Speculators checkpoint directly, `speculators_config.verifier.name_or_path` must be a remote model repository ID. References to local files or directories, including local paths that look like repository IDs, are rejected.
|
||||
|
||||
To use a local verifier, select it explicitly and configure the Speculators checkpoint as the draft model:
|
||||
|
||||
```bash
|
||||
vllm serve /path/to/verifier --speculative-config \
|
||||
'{"model": "/path/to/speculator", "method": "eagle3", "num_speculative_tokens": 1}'
|
||||
```
|
||||
|
||||
## Why use Speculators?
|
||||
|
||||
Large language models generate text one token at a time, which creates a fundamental bottleneck: each token requires a full forward pass through the model, leaving GPU compute underutilized while waiting for memory-bound operations.
|
||||
|
||||
@@ -10,11 +10,17 @@ runtime fields from the user are preserved while ``model`` stays auto-detected
|
||||
from the speculators format.
|
||||
"""
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.transformers_utils.config import maybe_override_with_speculators
|
||||
|
||||
_SPECULATORS_CONFIG_DICT = {
|
||||
_SPECULATORS_CONFIG_DICT: dict[str, Any] = {
|
||||
"architectures": ["DFlashDraftModel"],
|
||||
"speculators_config": {
|
||||
"algorithm": "dflash",
|
||||
@@ -154,3 +160,78 @@ def test_non_speculators_model_passes_user_config_through_unchanged():
|
||||
assert model == "fake-org/plain-llama"
|
||||
assert tok is None
|
||||
assert spec == user_overrides
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"verifier",
|
||||
[
|
||||
"/abs/path/to/other_tenant_private",
|
||||
"../private-model",
|
||||
"./private-model",
|
||||
"file:///private-model",
|
||||
"C:\\private-model",
|
||||
"",
|
||||
None,
|
||||
[],
|
||||
{},
|
||||
1,
|
||||
],
|
||||
)
|
||||
def test_speculators_rejects_invalid_verifier_references(verifier):
|
||||
config = deepcopy(_SPECULATORS_CONFIG_DICT)
|
||||
config["speculators_config"]["verifier"]["name_or_path"] = verifier
|
||||
with (
|
||||
_patched_get_config_dict(config),
|
||||
pytest.raises(ValueError, match="Speculators verifier must"),
|
||||
):
|
||||
maybe_override_with_speculators("fake-org/draft", None, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verifier", ["private-model", "fake-org/fake-verifier"])
|
||||
@pytest.mark.parametrize("symlink", [False, True])
|
||||
def test_speculators_rejects_local_verifiers_with_valid_repo_names(
|
||||
tmp_path, monkeypatch, verifier, symlink
|
||||
):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
local_verifier = tmp_path / verifier
|
||||
local_verifier.parent.mkdir(parents=True, exist_ok=True)
|
||||
if symlink:
|
||||
local_verifier.symlink_to(tmp_path / "private-target", target_is_directory=True)
|
||||
else:
|
||||
local_verifier.mkdir()
|
||||
|
||||
config = deepcopy(_SPECULATORS_CONFIG_DICT)
|
||||
config["speculators_config"]["verifier"]["name_or_path"] = verifier
|
||||
with (
|
||||
_patched_get_config_dict(config),
|
||||
pytest.raises(ValueError, match="Speculators verifier must"),
|
||||
):
|
||||
maybe_override_with_speculators("fake-org/draft", None, False)
|
||||
|
||||
|
||||
def test_non_speculators_local_model_remains_operator_selected(tmp_path):
|
||||
plain_config = {"architectures": ["LlamaForCausalLM"], "hidden_size": 64}
|
||||
with _patched_get_config_dict(plain_config):
|
||||
model, tokenizer, _ = maybe_override_with_speculators(
|
||||
str(tmp_path),
|
||||
str(tmp_path),
|
||||
False,
|
||||
vllm_speculative_config={"model": "fake-org/draft", "method": "eagle3"},
|
||||
)
|
||||
assert model == tokenizer == str(tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("engine_args_cls", [EngineArgs, AsyncEngineArgs])
|
||||
def test_speculators_rejects_local_verifier_before_model_loading(
|
||||
tmp_path, engine_args_cls
|
||||
):
|
||||
private_model = tmp_path / "private-model"
|
||||
private_model.mkdir()
|
||||
config = deepcopy(_SPECULATORS_CONFIG_DICT)
|
||||
config["speculators_config"]["verifier"]["name_or_path"] = str(private_model)
|
||||
draft = tmp_path / "draft"
|
||||
draft.mkdir()
|
||||
(draft / "config.json").write_text(json.dumps(config), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="Speculators verifier must"):
|
||||
engine_args_cls(model=str(draft)).create_engine_config()
|
||||
|
||||
@@ -14,6 +14,7 @@ import huggingface_hub
|
||||
import torch
|
||||
import transformers.configuration_utils as hf_configuration_utils
|
||||
from huggingface_hub import constants
|
||||
from huggingface_hub.utils import HFValidationError, validate_repo_id
|
||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
from transformers import GenerationConfig, PreTrainedConfig
|
||||
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
|
||||
@@ -844,6 +845,19 @@ def maybe_override_with_speculators(
|
||||
|
||||
# Override model and tokenizer with the verifier model from config
|
||||
verifier_model = speculators_config["verifier"]["name_or_path"]
|
||||
verifier_error = (
|
||||
"Speculators verifier must name a remote model repository. "
|
||||
"Select local verifiers explicitly with --model and --speculative-config."
|
||||
)
|
||||
if not isinstance(verifier_model, str):
|
||||
raise ValueError(verifier_error)
|
||||
try:
|
||||
validate_repo_id(verifier_model)
|
||||
except HFValidationError as exc:
|
||||
raise ValueError(verifier_error) from exc
|
||||
verifier_path = Path(verifier_model)
|
||||
if verifier_path.exists() or verifier_path.is_symlink():
|
||||
raise ValueError(verifier_error)
|
||||
model = tokenizer = verifier_model
|
||||
|
||||
return model, tokenizer, speculative_config
|
||||
|
||||
Reference in New Issue
Block a user