[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm import logger as vllm_logger
|
||||
from vllm.config import VllmConfig, WatermarkConfig
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
|
||||
def _input_processor(server_uses_watermarking: bool = True) -> InputProcessor:
|
||||
config = VllmConfig()
|
||||
config.model_config = SimpleNamespace(
|
||||
try_get_generation_config=lambda: {},
|
||||
logits_processors=None,
|
||||
return_sampling_mask=False,
|
||||
enable_trace_replay=True,
|
||||
is_multimodal_model=False,
|
||||
supports_multimodal_inputs=False,
|
||||
runner_type="generate",
|
||||
is_diffusion=False,
|
||||
)
|
||||
config.watermark_config = (
|
||||
WatermarkConfig(key=42) if server_uses_watermarking else None
|
||||
)
|
||||
renderer = MagicMock(spec=BaseRenderer)
|
||||
renderer.tokenizer = None
|
||||
renderer._executor = None
|
||||
return InputProcessor(config, renderer)
|
||||
|
||||
|
||||
def _validate(
|
||||
params: SamplingParams, server_uses_watermarking: bool = True
|
||||
) -> bool | None:
|
||||
processor = _input_processor(server_uses_watermarking)
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
return processor._validate_params(params, ("generate",))
|
||||
|
||||
|
||||
def _engine_core_request(params: SamplingParams) -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id="request",
|
||||
prompt_token_ids=[1],
|
||||
mm_features=None,
|
||||
sampling_params=params,
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_warning_once():
|
||||
vllm_logger._print_warning_once.cache_clear()
|
||||
yield
|
||||
vllm_logger._print_warning_once.cache_clear()
|
||||
|
||||
|
||||
def test_greedy_watermarked_requests_warn_once(caplog_vllm, reset_warning_once):
|
||||
first = SamplingParams(temperature=0)
|
||||
second = SamplingParams(temperature=0)
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
first_resolved = _validate(first)
|
||||
second_resolved = _validate(second)
|
||||
|
||||
message = "subsequent greedy requests will use ordinary greedy sampling"
|
||||
assert caplog_vllm.text.count(message) == 1
|
||||
assert first_resolved is False
|
||||
assert second_resolved is False
|
||||
# The caller's params objects are left untouched.
|
||||
assert first.watermarking is None
|
||||
assert second.watermarking is None
|
||||
|
||||
|
||||
def test_default_watermarking_resolves_when_configured():
|
||||
enabled = SamplingParams()
|
||||
|
||||
assert _validate(enabled) is True
|
||||
# The caller's params object keeps inheriting.
|
||||
assert enabled.watermarking is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("watermarking", [True])
|
||||
def test_requested_watermarking_without_engine_config_warns_and_disables(
|
||||
watermarking, caplog_vllm, reset_warning_once
|
||||
):
|
||||
params = SamplingParams(watermarking=watermarking)
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
resolved = _validate(params, server_uses_watermarking=False)
|
||||
|
||||
assert "engine has no watermark configuration" in caplog_vllm.text
|
||||
assert resolved is False
|
||||
assert params.watermarking is watermarking
|
||||
|
||||
|
||||
@pytest.mark.parametrize("watermarking", [None, True])
|
||||
def test_explicit_and_default_greedy_warn_and_disable(
|
||||
watermarking, caplog_vllm, reset_warning_once
|
||||
):
|
||||
params = SamplingParams(temperature=0, watermarking=watermarking)
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
resolved = _validate(params)
|
||||
|
||||
assert "ordinary greedy sampling" in caplog_vllm.text
|
||||
assert resolved is False
|
||||
assert params.watermarking is watermarking
|
||||
|
||||
|
||||
def test_trace_replay_with_watermarking_warns_and_disables(
|
||||
caplog_vllm, reset_warning_once
|
||||
):
|
||||
params = SamplingParams(trace_decode_token_ids=[1])
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
resolved = _validate(params)
|
||||
|
||||
assert "trace replay requests will run without watermarking" in caplog_vllm.text
|
||||
assert resolved is False
|
||||
assert params.watermarking is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
SamplingParams(temperature=0, watermarking=False),
|
||||
SamplingParams(trace_decode_token_ids=[1], watermarking=False),
|
||||
],
|
||||
)
|
||||
def test_incompatible_modes_are_allowed_when_watermarking_is_disabled(params):
|
||||
_validate(params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
SamplingParams(),
|
||||
SamplingParams(temperature=0),
|
||||
SamplingParams(trace_decode_token_ids=[1]),
|
||||
],
|
||||
)
|
||||
def test_requests_without_engine_config_are_unwatermarked(
|
||||
params, caplog_vllm, reset_warning_once
|
||||
):
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
resolved = _validate(params, server_uses_watermarking=False)
|
||||
|
||||
assert "engine has no watermark configuration" not in caplog_vllm.text
|
||||
assert resolved is False
|
||||
assert params.watermarking is None
|
||||
|
||||
|
||||
def test_trace_replay_takes_precedence_over_greedy_warning(
|
||||
caplog_vllm, reset_warning_once
|
||||
):
|
||||
params = SamplingParams(temperature=0, trace_decode_token_ids=[1])
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
_validate(params)
|
||||
|
||||
assert "trace replay requests will run without watermarking" in caplog_vllm.text
|
||||
assert "ordinary greedy sampling" not in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
SamplingParams(seed=42),
|
||||
SamplingParams(n=2),
|
||||
SamplingParams(n=2, seed=42),
|
||||
],
|
||||
)
|
||||
def test_seeded_and_parallel_sampling_are_allowed(params):
|
||||
_validate(params)
|
||||
|
||||
|
||||
def test_direct_engine_trace_replay_warns_and_disables(
|
||||
caplog_vllm, monkeypatch, reset_warning_once
|
||||
):
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.input_processor = _input_processor()
|
||||
engine.vllm_config = engine.input_processor.vllm_config
|
||||
assign_request_id = MagicMock()
|
||||
monkeypatch.setattr(engine.input_processor, "assign_request_id", assign_request_id)
|
||||
engine.output_processor = MagicMock()
|
||||
engine.engine_core = MagicMock()
|
||||
request = _engine_core_request(SamplingParams(trace_decode_token_ids=[1]))
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
engine.add_request(
|
||||
request.request_id,
|
||||
request,
|
||||
SamplingParams(watermarking=False),
|
||||
)
|
||||
|
||||
assert "trace replay requests will run without watermarking" in caplog_vllm.text
|
||||
assert request.sampling_params.watermarking is False
|
||||
assert request.sampling_params._watermarking_skipped is True
|
||||
assign_request_id.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_direct_engine_request_uses_embedded_opt_out(monkeypatch):
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.input_processor = _input_processor()
|
||||
engine.vllm_config = engine.input_processor.vllm_config
|
||||
assign_request_id = MagicMock()
|
||||
monkeypatch.setattr(engine.input_processor, "assign_request_id", assign_request_id)
|
||||
engine.output_processor = MagicMock()
|
||||
engine.engine_core = MagicMock()
|
||||
request = _engine_core_request(SamplingParams(temperature=0, watermarking=False))
|
||||
|
||||
engine.add_request(
|
||||
request.request_id,
|
||||
request,
|
||||
SamplingParams(temperature=0),
|
||||
)
|
||||
|
||||
assert request.sampling_params._watermarking_skipped is False
|
||||
assign_request_id.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_direct_async_engine_trace_replay_warns_and_disables(
|
||||
caplog_vllm, monkeypatch, reset_warning_once
|
||||
):
|
||||
engine = object.__new__(AsyncLLM)
|
||||
engine.engine_core = SimpleNamespace(
|
||||
resources=SimpleNamespace(engine_dead=False),
|
||||
shutdown=lambda **kwargs: None,
|
||||
add_request_async=AsyncMock(),
|
||||
)
|
||||
engine.output_handler = None
|
||||
engine.input_processor = _input_processor()
|
||||
engine.vllm_config = engine.input_processor.vllm_config
|
||||
engine.output_processor = MagicMock()
|
||||
engine.log_requests = False
|
||||
monkeypatch.setattr(engine, "_run_output_handler", MagicMock())
|
||||
request = _engine_core_request(SamplingParams(trace_decode_token_ids=[1]))
|
||||
|
||||
async def add_request() -> None:
|
||||
await engine.add_request(
|
||||
request.request_id,
|
||||
request,
|
||||
SamplingParams(watermarking=False),
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
asyncio.run(add_request())
|
||||
|
||||
assert "trace replay requests will run without watermarking" in caplog_vllm.text
|
||||
assert request.sampling_params.watermarking is False
|
||||
assert request.sampling_params._watermarking_skipped is True
|
||||
|
||||
|
||||
def test_direct_async_engine_greedy_request_warns_and_disables_watermarking(
|
||||
caplog_vllm, monkeypatch, reset_warning_once
|
||||
):
|
||||
engine = object.__new__(AsyncLLM)
|
||||
engine.engine_core = SimpleNamespace(
|
||||
resources=SimpleNamespace(engine_dead=False),
|
||||
shutdown=lambda **kwargs: None,
|
||||
add_request_async=AsyncMock(),
|
||||
)
|
||||
engine.output_handler = None
|
||||
engine.input_processor = _input_processor()
|
||||
engine.vllm_config = engine.input_processor.vllm_config
|
||||
engine.output_processor = MagicMock()
|
||||
engine.log_requests = False
|
||||
monkeypatch.setattr(engine, "_run_output_handler", MagicMock())
|
||||
request = _engine_core_request(SamplingParams(temperature=0))
|
||||
|
||||
async def add_request() -> None:
|
||||
await engine.add_request(
|
||||
request.request_id,
|
||||
request,
|
||||
SamplingParams(watermarking=False),
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
asyncio.run(add_request())
|
||||
|
||||
assert "subsequent greedy requests will use ordinary greedy sampling" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
assert request.sampling_params.watermarking is False
|
||||
assert request.sampling_params._watermarking_skipped is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "server_uses_watermarking", "skipped"),
|
||||
[
|
||||
({}, True, False),
|
||||
({"temperature": 0}, True, True),
|
||||
({"temperature": 0, "watermarking": False}, True, False),
|
||||
({"temperature": 0}, False, False),
|
||||
],
|
||||
)
|
||||
def test_apply_watermarking_records_skipped_requests(
|
||||
kwargs, server_uses_watermarking, skipped, reset_warning_once
|
||||
):
|
||||
params = SamplingParams(**kwargs)
|
||||
processor = _input_processor(server_uses_watermarking)
|
||||
processor.apply_watermarking(params, processor.resolve_watermarking(params))
|
||||
assert params._watermarking_skipped is skipped
|
||||
|
||||
# Resolving an already resolved request keeps the flag.
|
||||
processor.apply_watermarking(params, processor.resolve_watermarking(params))
|
||||
assert params._watermarking_skipped is skipped
|
||||
|
||||
|
||||
def test_shared_sampling_params_are_not_rewritten_by_admission(reset_warning_once):
|
||||
"""A params object reused across prompts/engines must keep inheriting."""
|
||||
params = SamplingParams(temperature=0.8)
|
||||
|
||||
assert _validate(params, server_uses_watermarking=False) is False
|
||||
assert params.watermarking is None
|
||||
# Same object submitted to a watermarked engine must still be watermarked.
|
||||
assert _validate(params, server_uses_watermarking=True) is True
|
||||
|
||||
greedy = SamplingParams(temperature=0.0)
|
||||
assert _validate(greedy) is False
|
||||
greedy.temperature = 0.8
|
||||
assert _validate(greedy) is True
|
||||
|
||||
|
||||
def test_shared_beam_search_params_are_not_rewritten_by_admission(
|
||||
caplog_vllm, reset_warning_once
|
||||
):
|
||||
"""Beam search must keep resolving per engine after reuse."""
|
||||
params = BeamSearchParams(beam_width=2, max_tokens=4, temperature=0.8)
|
||||
|
||||
# A plain engine leaves the tri-state unresolved on the caller's object.
|
||||
assert _input_processor(False).resolve_watermarking(params) is False
|
||||
assert params.watermarking is None
|
||||
|
||||
# The same object on a watermarked engine is warned about and still
|
||||
# resolves to False, without the first decision being written back.
|
||||
with caplog_vllm.at_level("WARNING"):
|
||||
assert _input_processor(True).resolve_watermarking(params) is False
|
||||
assert "beam search requests will run without watermarking" in caplog_vllm.text
|
||||
assert params.watermarking is None
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
MODEL = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
MAX_NUM_BATCHED_TOKENS = 32
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def watermarked_llm(vllm_runner) -> Generator[LLM, None, None]:
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS,
|
||||
max_model_len=128,
|
||||
max_num_seqs=4,
|
||||
disable_log_stats=False,
|
||||
watermark_config={"algorithm": "gumbel", "key": 42},
|
||||
) as runner:
|
||||
yield runner.llm
|
||||
|
||||
|
||||
def test_watermark_with_combined_sampling_controls(watermarked_llm: LLM):
|
||||
params = SamplingParams(
|
||||
temperature=0.8,
|
||||
top_k=8,
|
||||
top_p=0.9,
|
||||
min_p=0.01,
|
||||
presence_penalty=0.2,
|
||||
frequency_penalty=0.2,
|
||||
repetition_penalty=1.1,
|
||||
logit_bias={10: 100.0},
|
||||
allowed_token_ids=[10],
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
)
|
||||
|
||||
output = watermarked_llm.generate("Sampling smoke test", params)[0]
|
||||
|
||||
assert list(output.outputs[0].token_ids) == [10] * 8
|
||||
|
||||
# The assertion above holds with or without watermarking, because
|
||||
# allowed_token_ids leaves a single candidate. Repeat with a wider allowed
|
||||
# set, where only watermarked sampling is reproducible without a seed.
|
||||
wide = SamplingParams(
|
||||
temperature=0.8,
|
||||
top_k=8,
|
||||
top_p=0.9,
|
||||
min_p=0.01,
|
||||
presence_penalty=0.2,
|
||||
frequency_penalty=0.2,
|
||||
repetition_penalty=1.1,
|
||||
allowed_token_ids=list(range(10, 210)),
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
)
|
||||
|
||||
first = watermarked_llm.generate("Sampling smoke test", wide)[0]
|
||||
second = watermarked_llm.generate("Sampling smoke test", wide)[0]
|
||||
|
||||
assert set(first.outputs[0].token_ids) <= set(range(10, 210))
|
||||
assert first.outputs[0].token_ids == second.outputs[0].token_ids
|
||||
|
||||
|
||||
def test_watermark_with_penalties_and_nucleus_sampling(watermarked_llm: LLM):
|
||||
params = SamplingParams(
|
||||
temperature=0.8,
|
||||
top_k=8,
|
||||
top_p=0.9,
|
||||
min_p=0.01,
|
||||
presence_penalty=0.2,
|
||||
frequency_penalty=0.2,
|
||||
repetition_penalty=1.1,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
)
|
||||
|
||||
first = watermarked_llm.generate("Penalty smoke test", params)[0]
|
||||
second = watermarked_llm.generate("Penalty smoke test", params)[0]
|
||||
|
||||
assert len(first.outputs[0].token_ids) == 8
|
||||
assert first.outputs[0].token_ids == second.outputs[0].token_ids
|
||||
|
||||
|
||||
def test_plain_gumbel_seeded_parallel_sampling_is_deterministic(
|
||||
watermarked_llm: LLM,
|
||||
):
|
||||
output = watermarked_llm.generate(
|
||||
"Seeded parallel sampling smoke test",
|
||||
SamplingParams(
|
||||
n=2,
|
||||
seed=42,
|
||||
temperature=0.8,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
),
|
||||
)[0]
|
||||
|
||||
assert len(output.outputs) == 2
|
||||
assert all(len(candidate.token_ids) == 8 for candidate in output.outputs)
|
||||
assert output.outputs[0].token_ids == output.outputs[1].token_ids
|
||||
|
||||
# Plain Gumbel keys on the token context, so the request seed does not
|
||||
# change a watermarked draw. Pin that down rather than implying the seed
|
||||
# is what makes the candidates match.
|
||||
unseeded = watermarked_llm.generate(
|
||||
"Seeded parallel sampling smoke test",
|
||||
SamplingParams(
|
||||
n=2,
|
||||
temperature=0.8,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
),
|
||||
)[0]
|
||||
|
||||
assert unseeded.outputs[0].token_ids == output.outputs[0].token_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"structured_outputs",
|
||||
[
|
||||
StructuredOutputsParams(choice=["A", "B"]),
|
||||
StructuredOutputsParams(grammar='root ::= "A" | "B"'),
|
||||
],
|
||||
ids=["choice", "grammar"],
|
||||
)
|
||||
def test_watermark_with_structured_output(
|
||||
watermarked_llm: LLM, structured_outputs: StructuredOutputsParams
|
||||
):
|
||||
params = SamplingParams(
|
||||
temperature=0.8,
|
||||
max_tokens=8,
|
||||
structured_outputs=structured_outputs,
|
||||
)
|
||||
|
||||
output = watermarked_llm.generate("Choose A or B:", params)[0]
|
||||
|
||||
assert output.outputs[0].text.strip() in ("A", "B")
|
||||
|
||||
|
||||
def test_watermark_with_wide_grammar_is_deterministic(watermarked_llm: LLM):
|
||||
# A wider grammar makes deterministic watermarking observable.
|
||||
params = SamplingParams(
|
||||
temperature=0.8,
|
||||
max_tokens=8,
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
grammar="root ::= [A-Z] [A-Z] [A-Z] [A-Z] [A-Z] [A-Z]"
|
||||
),
|
||||
)
|
||||
|
||||
first = watermarked_llm.generate("Emit six letters:", params)[0]
|
||||
second = watermarked_llm.generate("Emit six letters:", params)[0]
|
||||
|
||||
assert first.outputs[0].text.strip().isupper()
|
||||
assert first.outputs[0].token_ids == second.outputs[0].token_ids
|
||||
|
||||
|
||||
def test_watermark_with_prefix_cache_and_chunked_prefill(watermarked_llm: LLM):
|
||||
prompt = {"prompt_token_ids": [101] * 48}
|
||||
params = SamplingParams(temperature=0.8, max_tokens=8, ignore_eos=True)
|
||||
|
||||
assert len(prompt["prompt_token_ids"]) > MAX_NUM_BATCHED_TOKENS
|
||||
|
||||
cold = watermarked_llm.generate(prompt, params)[0]
|
||||
cached = watermarked_llm.generate(prompt, params)[0]
|
||||
|
||||
assert cached.num_cached_tokens > 0
|
||||
assert cold.outputs[0].token_ids == cached.outputs[0].token_ids
|
||||
|
||||
|
||||
def test_mixed_watermarked_batch(watermarked_llm: LLM):
|
||||
prompts = [
|
||||
{"prompt_token_ids": [101] * 32 + [102]},
|
||||
{"prompt_token_ids": [201] * 32 + [202]},
|
||||
]
|
||||
marked = SamplingParams(temperature=0.8, max_tokens=8, ignore_eos=True)
|
||||
ordinary = SamplingParams(
|
||||
temperature=0.8,
|
||||
seed=123,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
watermarking=False,
|
||||
)
|
||||
|
||||
batched = watermarked_llm.generate(prompts, [marked, ordinary])
|
||||
marked_alone = watermarked_llm.generate(prompts[0], marked)[0]
|
||||
ordinary_alone = watermarked_llm.generate(prompts[1], ordinary)[0]
|
||||
|
||||
assert batched[0].outputs[0].token_ids == marked_alone.outputs[0].token_ids
|
||||
assert batched[1].outputs[0].token_ids == ordinary_alone.outputs[0].token_ids
|
||||
@@ -1,284 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking import (
|
||||
DualKeyGumbelWatermarkDetector,
|
||||
DualKeyGumbelWatermarker,
|
||||
GumbelWatermarkDetector,
|
||||
GumbelWatermarker,
|
||||
derive_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
|
||||
from vllm.v1.watermarking.watermarker import RandomSampler
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
|
||||
|
||||
def test_gamma_survival_integer_shape():
|
||||
assert _gamma_survival_integer_shape(0.0, 4) == 1.0
|
||||
assert _gamma_survival_integer_shape(2.0, 1) == pytest.approx(0.1353352832)
|
||||
|
||||
|
||||
def test_detector_deduplicates_repeated_prf_inputs_by_default():
|
||||
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 1, 1, 1, 1])
|
||||
|
||||
assert detection.num_scored_tokens == 2
|
||||
|
||||
|
||||
def test_detector_can_score_repeated_prf_inputs():
|
||||
detection = GumbelWatermarkDetector(
|
||||
key=42, context_width=1, deduplicate_contexts=False
|
||||
).detect([1, 1, 1, 1, 1])
|
||||
|
||||
assert detection.num_scored_tokens == 5
|
||||
|
||||
|
||||
def test_detector_deduplicates_context_even_when_target_differs():
|
||||
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 2, 1, 3])
|
||||
|
||||
assert detection.num_scored_tokens == 3
|
||||
|
||||
|
||||
def test_watermarker_respects_filtered_token_support():
|
||||
logits = torch.full((2, 8), -torch.inf)
|
||||
logits[0, 3] = 0
|
||||
logits[1, 6] = 0
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
|
||||
token_ids = (
|
||||
GumbelWatermarker(key=42)
|
||||
.sample(logits, contexts, lambda values: None)
|
||||
.token_ids
|
||||
)
|
||||
|
||||
assert torch.equal(token_ids, torch.tensor([3, 6]))
|
||||
|
||||
|
||||
def test_dual_key_detector_scores_each_token_against_both_keys():
|
||||
token_ids = [1, 2, 3, 4, 5]
|
||||
key_a_detector = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_a"),
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
)
|
||||
key_b_detector = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_b"),
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
)
|
||||
detector = DualKeyGumbelWatermarkDetector(
|
||||
key=42,
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
alpha=0.5,
|
||||
)
|
||||
|
||||
key_a = key_a_detector.detect(token_ids)
|
||||
key_b = key_b_detector.detect(token_ids)
|
||||
dual = detector.detect(token_ids)
|
||||
|
||||
assert dual.score == pytest.approx((key_a.score + key_b.score) / 2)
|
||||
assert dual.p_value == pytest.approx(
|
||||
_gamma_survival_integer_shape(dual.score * 2, dual.num_scored_tokens * 2)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha", [0.2, 0.7])
|
||||
def test_dual_key_detector_weights_and_recalibrates_scores(alpha: float):
|
||||
token_ids = [1, 2, 3, 4, 5]
|
||||
key_a = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_a"), deduplicate_contexts=False
|
||||
).detect(token_ids)
|
||||
key_b = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_b"), deduplicate_contexts=False
|
||||
).detect(token_ids)
|
||||
dual = DualKeyGumbelWatermarkDetector(
|
||||
key=42, deduplicate_contexts=False, alpha=alpha
|
||||
).detect(token_ids)
|
||||
|
||||
variance = (1 - alpha) ** 2 + alpha**2
|
||||
expected_p_value = torch.special.gammaincc(
|
||||
torch.tensor(dual.num_scored_tokens / variance, dtype=torch.float64),
|
||||
torch.tensor(dual.score / variance, dtype=torch.float64),
|
||||
).item()
|
||||
|
||||
assert dual.score == pytest.approx((1 - alpha) * key_a.score + alpha * key_b.score)
|
||||
assert dual.p_value == pytest.approx(expected_p_value)
|
||||
|
||||
|
||||
def test_dual_key_detector_rejects_invalid_alpha():
|
||||
with pytest.raises(ValueError, match="alpha must be between 0 and 1"):
|
||||
DualKeyGumbelWatermarkDetector(key=42, alpha=1.1)
|
||||
|
||||
|
||||
def test_dual_key_alpha_defaults_match():
|
||||
config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
watermarker = DualKeyGumbelWatermarker(key=42)
|
||||
detector = DualKeyGumbelWatermarkDetector(key=42)
|
||||
|
||||
assert detector.alpha == watermarker.alpha == config.alpha
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("key", [1, 2**32, 42, 15726070495360670683])
|
||||
@pytest.mark.parametrize("context_width", [1, 4, 16])
|
||||
def test_fused_watermarker_matches_cpu(key: int, context_width: int):
|
||||
torch.manual_seed(0)
|
||||
contexts = torch.randint(0, 248320, (32, context_width), dtype=torch.int64)
|
||||
logits = torch.randn(32, 8193)
|
||||
watermarker = GumbelWatermarker(key, context_width)
|
||||
|
||||
expected = watermarker.sample(logits, contexts).token_ids
|
||||
actual = watermarker.sample(logits.cuda(), contexts.cuda()).token_ids.cpu()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_fused_watermarker_handles_nan_logits():
|
||||
contexts = torch.zeros((2, 4), dtype=torch.int64, device="cuda")
|
||||
logits = torch.full((2, 1025), float("nan"), device="cuda")
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
token_ids = watermarker.sample(logits, contexts).token_ids
|
||||
|
||||
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_fused_watermarker_handles_noncontiguous_inputs():
|
||||
contexts = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")[
|
||||
:, ::2
|
||||
]
|
||||
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
expected = watermarker.sample(logits.contiguous(), contexts.contiguous()).token_ids
|
||||
actual = watermarker.sample(logits, contexts).token_ids
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool):
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(8, 8193, device="cuda")
|
||||
context_storage = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")
|
||||
contexts = context_storage[:, ::2]
|
||||
repeated_mask = torch.tensor(
|
||||
[False, False, False, True, False, False, True, False], device="cuda"
|
||||
)
|
||||
watermarking = torch.tensor(
|
||||
[True, False, True, True, False, True, True, False], device="cuda"
|
||||
)
|
||||
req_indices = torch.tensor([3, 1, 7, 0, 4, 2, 6, 5], device="cuda")
|
||||
temperatures = torch.ones(8, dtype=torch.float32, device="cuda")
|
||||
temperatures[2] = 0
|
||||
seeds = torch.arange(8, dtype=torch.int64, device="cuda") + 1000
|
||||
positions = torch.arange(8, dtype=torch.int64, device="cuda") + 100
|
||||
watermark_mask = (
|
||||
watermarking[req_indices] & (temperatures[req_indices] != 0) & ~repeated_mask
|
||||
)
|
||||
|
||||
watermarked = philox_gumbel_sample(logits, contexts, 42)
|
||||
ordinary = gumbel_sample(
|
||||
logits,
|
||||
req_indices,
|
||||
temperatures,
|
||||
seeds,
|
||||
positions,
|
||||
apply_temperature=False,
|
||||
is_drafting=False,
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
expected = torch.where(watermark_mask, watermarked, ordinary)
|
||||
actual = philox_gumbel_sample(
|
||||
logits,
|
||||
contexts,
|
||||
42,
|
||||
skip_mask=~watermark_mask,
|
||||
expanded_idx_mapping=req_indices,
|
||||
temperatures=temperatures,
|
||||
seeds=seeds,
|
||||
positions=positions,
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
|
||||
torch.manual_seed(0)
|
||||
num_rows, vocab_size, num_steps = 5, 4099, 3
|
||||
logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16)
|
||||
temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda")
|
||||
processed_logits = logits / torch.where(
|
||||
temperatures == 0, 1, temperatures
|
||||
).unsqueeze(-1)
|
||||
contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda")
|
||||
idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda")
|
||||
seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10
|
||||
positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20
|
||||
# The skipped rows must divide by a temperature other than 1, or the
|
||||
# unprocessed logits the cache stores are the logits the sampler sees.
|
||||
# Row 0 keeps a greedy row in the skipped set.
|
||||
skip_mask = torch.tensor([True, True, False, False, True], device="cuda")
|
||||
cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps
|
||||
cache = torch.zeros(
|
||||
num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda"
|
||||
)
|
||||
|
||||
ordinary = gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperatures,
|
||||
seeds,
|
||||
positions,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
)
|
||||
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
|
||||
expected = torch.where(skip_mask, ordinary, watermarked)
|
||||
actual = (
|
||||
GumbelWatermarker(key=42, context_width=4)
|
||||
.sample(
|
||||
processed_logits,
|
||||
contexts,
|
||||
random_sampler=RandomSampler(
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
temperatures=temperatures,
|
||||
seeds=seeds,
|
||||
positions=positions,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
logits_cache_source=logits,
|
||||
),
|
||||
skip_mask=skip_mask,
|
||||
)
|
||||
.token_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size]
|
||||
assert torch.equal(stored.view(torch.int16), logits.view(torch.int16))
|
||||
assert not cache[:, :, -1].any()
|
||||
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"parallel_kwargs",
|
||||
[
|
||||
{"tensor_parallel_size": 2, "enable_batch_sharded_sampling": True},
|
||||
{"pipeline_parallel_size": 2},
|
||||
],
|
||||
ids=["tp2-sharded-sampling", "pp2"],
|
||||
)
|
||||
def test_watermarked_generation_with_model_parallelism(vllm_runner, parallel_kwargs):
|
||||
prompts = ["Parallel watermark smoke"] * 2
|
||||
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
max_model_len=128,
|
||||
max_num_seqs=4,
|
||||
watermark_config={"algorithm": "gumbel", "key": 42},
|
||||
**parallel_kwargs,
|
||||
) as runner:
|
||||
first = runner.llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=0.8,
|
||||
seed=1,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
),
|
||||
)
|
||||
second = runner.llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=0.8,
|
||||
seed=2,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
first_tokens = [output.outputs[0].token_ids for output in first]
|
||||
second_tokens = [output.outputs[0].token_ids for output in second]
|
||||
assert all(len(token_ids) == 8 for token_ids in first_tokens)
|
||||
assert first_tokens[0] == first_tokens[1]
|
||||
assert first_tokens == second_tokens
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_watermarked_generation_with_data_parallelism():
|
||||
async def run() -> None:
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
AsyncEngineArgs(
|
||||
model=MODEL,
|
||||
dtype="half",
|
||||
data_parallel_size=2,
|
||||
enforce_eager=True,
|
||||
max_model_len=128,
|
||||
max_num_seqs=4,
|
||||
watermark_config={"algorithm": "gumbel", "key": 42},
|
||||
)
|
||||
)
|
||||
|
||||
async def generate(index: int) -> tuple[int, ...]:
|
||||
final_output = None
|
||||
async for output in engine.generate(
|
||||
"Data-parallel watermark smoke",
|
||||
SamplingParams(
|
||||
temperature=0.8,
|
||||
seed=index,
|
||||
max_tokens=8,
|
||||
ignore_eos=True,
|
||||
),
|
||||
request_id=f"watermark-dp-{index}",
|
||||
data_parallel_rank=index,
|
||||
):
|
||||
final_output = output
|
||||
assert final_output is not None
|
||||
return tuple(final_output.outputs[0].token_ids)
|
||||
|
||||
try:
|
||||
token_ids = await asyncio.gather(generate(0), generate(1))
|
||||
assert len(token_ids[0]) == 8
|
||||
assert token_ids[0] == token_ids[1]
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
try:
|
||||
asyncio.run(run())
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -1,44 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF
|
||||
|
||||
|
||||
def test_philox_compatibility_vector():
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
token_ids = torch.tensor([7, 8])
|
||||
expected_mantissas = [[1661833, 10571167], [9742331, 13375724]]
|
||||
expected = (
|
||||
(torch.tensor(expected_mantissas, dtype=torch.float64) + 1) / (2**24 + 1)
|
||||
).to(torch.float32)
|
||||
|
||||
assert torch.equal(PhiloxPRF(42).uniform(contexts, token_ids), expected)
|
||||
|
||||
|
||||
def test_prf_pairs_contexts_with_target_tokens():
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
token_ids = torch.tensor([[7], [8]])
|
||||
|
||||
assert PhiloxPRF(42).uniform(contexts, token_ids).shape == (2, 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("key", [42, 15726070495360670683])
|
||||
@pytest.mark.parametrize("context_width", [1, 4, 16])
|
||||
def test_philox_accelerator_matches_cpu(key: int, context_width: int):
|
||||
contexts = torch.arange(2 * context_width, dtype=torch.int64).reshape(
|
||||
2, context_width
|
||||
)
|
||||
token_ids = torch.arange(1024, dtype=torch.int64)
|
||||
prf = PhiloxPRF(key)
|
||||
|
||||
expected = prf.uniform(contexts, token_ids)
|
||||
actual = prf.uniform(contexts.cuda(), token_ids.cuda()).cpu()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
from vllm.v1.watermarking import GumbelWatermarkDetector
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_llm_generated_sequence_is_watermarked(vllm_runner):
|
||||
"""Engine-level watermarking must produce tokens the detector recognizes."""
|
||||
watermark_config = {"key": 42, "context_width": 4}
|
||||
detector = GumbelWatermarkDetector(**watermark_config)
|
||||
|
||||
runner = vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
watermark_config=watermark_config,
|
||||
seed=0,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.2,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
request = "Tell me a story about an explorer who discovers a mysterious island."
|
||||
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
|
||||
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
|
||||
params_greedy = SamplingParams(temperature=0.0, max_tokens=16)
|
||||
|
||||
with runner:
|
||||
output_use_wm = runner.llm.generate(request, params_use_wm)
|
||||
output_no_wm = runner.llm.generate(request, params_no_wm)
|
||||
runner.llm.generate(request, params_greedy)
|
||||
metrics = runner.llm.get_metrics()
|
||||
|
||||
# The greedy request is skipped; the opt-out one is not watermarked.
|
||||
counters = [m for m in metrics if isinstance(m, Counter)]
|
||||
status = {
|
||||
m.labels["status"]: m.value
|
||||
for m in counters
|
||||
if m.name == "vllm:watermark_requests"
|
||||
}
|
||||
assert status == {"watermarked": 1, "skipped": 1, "not_watermarked": 1}
|
||||
num_success = sum(m.value for m in counters if m.name == "vllm:request_success")
|
||||
assert sum(status.values()) == num_success
|
||||
|
||||
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
|
||||
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
|
||||
|
||||
assert result_use_wm.is_watermarked, result_use_wm
|
||||
recorded_p_value = 9.83e-20
|
||||
assert result_use_wm.p_value < recorded_p_value * 10 # tolerance
|
||||
|
||||
assert not result_no_wm.is_watermarked, result_no_wm
|
||||
Reference in New Issue
Block a user