[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-354
View File
@@ -1,354 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from vllm import SamplingParams
from vllm import logger as vllm_logger
from vllm.config import VllmConfig, WatermarkConfig
from vllm.renderers import BaseRenderer
from vllm.sampling_params import BeamSearchParams
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.input_processor import InputProcessor
from vllm.v1.engine.llm_engine import LLMEngine
def _input_processor(server_uses_watermarking: bool = True) -> InputProcessor:
config = VllmConfig()
config.model_config = SimpleNamespace(
try_get_generation_config=lambda: {},
logits_processors=None,
return_sampling_mask=False,
enable_trace_replay=True,
is_multimodal_model=False,
supports_multimodal_inputs=False,
runner_type="generate",
is_diffusion=False,
)
config.watermark_config = (
WatermarkConfig(key=42) if server_uses_watermarking else None
)
renderer = MagicMock(spec=BaseRenderer)
renderer.tokenizer = None
renderer._executor = None
return InputProcessor(config, renderer)
def _validate(
params: SamplingParams, server_uses_watermarking: bool = True
) -> bool | None:
processor = _input_processor(server_uses_watermarking)
with patch.object(SamplingParams, "verify"):
return processor._validate_params(params, ("generate",))
def _engine_core_request(params: SamplingParams) -> EngineCoreRequest:
return EngineCoreRequest(
request_id="request",
prompt_token_ids=[1],
mm_features=None,
sampling_params=params,
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
@pytest.fixture
def reset_warning_once():
vllm_logger._print_warning_once.cache_clear()
yield
vllm_logger._print_warning_once.cache_clear()
def test_greedy_watermarked_requests_warn_once(caplog_vllm, reset_warning_once):
first = SamplingParams(temperature=0)
second = SamplingParams(temperature=0)
with caplog_vllm.at_level("WARNING"):
first_resolved = _validate(first)
second_resolved = _validate(second)
message = "subsequent greedy requests will use ordinary greedy sampling"
assert caplog_vllm.text.count(message) == 1
assert first_resolved is False
assert second_resolved is False
# The caller's params objects are left untouched.
assert first.watermarking is None
assert second.watermarking is None
def test_default_watermarking_resolves_when_configured():
enabled = SamplingParams()
assert _validate(enabled) is True
# The caller's params object keeps inheriting.
assert enabled.watermarking is None
@pytest.mark.parametrize("watermarking", [True])
def test_requested_watermarking_without_engine_config_warns_and_disables(
watermarking, caplog_vllm, reset_warning_once
):
params = SamplingParams(watermarking=watermarking)
with caplog_vllm.at_level("WARNING"):
resolved = _validate(params, server_uses_watermarking=False)
assert "engine has no watermark configuration" in caplog_vllm.text
assert resolved is False
assert params.watermarking is watermarking
@pytest.mark.parametrize("watermarking", [None, True])
def test_explicit_and_default_greedy_warn_and_disable(
watermarking, caplog_vllm, reset_warning_once
):
params = SamplingParams(temperature=0, watermarking=watermarking)
with caplog_vllm.at_level("WARNING"):
resolved = _validate(params)
assert "ordinary greedy sampling" in caplog_vllm.text
assert resolved is False
assert params.watermarking is watermarking
def test_trace_replay_with_watermarking_warns_and_disables(
caplog_vllm, reset_warning_once
):
params = SamplingParams(trace_decode_token_ids=[1])
with caplog_vllm.at_level("WARNING"):
resolved = _validate(params)
assert "trace replay requests will run without watermarking" in caplog_vllm.text
assert resolved is False
assert params.watermarking is None
@pytest.mark.parametrize(
"params",
[
SamplingParams(temperature=0, watermarking=False),
SamplingParams(trace_decode_token_ids=[1], watermarking=False),
],
)
def test_incompatible_modes_are_allowed_when_watermarking_is_disabled(params):
_validate(params)
@pytest.mark.parametrize(
"params",
[
SamplingParams(),
SamplingParams(temperature=0),
SamplingParams(trace_decode_token_ids=[1]),
],
)
def test_requests_without_engine_config_are_unwatermarked(
params, caplog_vllm, reset_warning_once
):
with caplog_vllm.at_level("WARNING"):
resolved = _validate(params, server_uses_watermarking=False)
assert "engine has no watermark configuration" not in caplog_vllm.text
assert resolved is False
assert params.watermarking is None
def test_trace_replay_takes_precedence_over_greedy_warning(
caplog_vllm, reset_warning_once
):
params = SamplingParams(temperature=0, trace_decode_token_ids=[1])
with caplog_vllm.at_level("WARNING"):
_validate(params)
assert "trace replay requests will run without watermarking" in caplog_vllm.text
assert "ordinary greedy sampling" not in caplog_vllm.text
@pytest.mark.parametrize(
"params",
[
SamplingParams(seed=42),
SamplingParams(n=2),
SamplingParams(n=2, seed=42),
],
)
def test_seeded_and_parallel_sampling_are_allowed(params):
_validate(params)
def test_direct_engine_trace_replay_warns_and_disables(
caplog_vllm, monkeypatch, reset_warning_once
):
engine = object.__new__(LLMEngine)
engine.input_processor = _input_processor()
engine.vllm_config = engine.input_processor.vllm_config
assign_request_id = MagicMock()
monkeypatch.setattr(engine.input_processor, "assign_request_id", assign_request_id)
engine.output_processor = MagicMock()
engine.engine_core = MagicMock()
request = _engine_core_request(SamplingParams(trace_decode_token_ids=[1]))
with caplog_vllm.at_level("WARNING"):
engine.add_request(
request.request_id,
request,
SamplingParams(watermarking=False),
)
assert "trace replay requests will run without watermarking" in caplog_vllm.text
assert request.sampling_params.watermarking is False
assert request.sampling_params._watermarking_skipped is True
assign_request_id.assert_called_once_with(request)
def test_direct_engine_request_uses_embedded_opt_out(monkeypatch):
engine = object.__new__(LLMEngine)
engine.input_processor = _input_processor()
engine.vllm_config = engine.input_processor.vllm_config
assign_request_id = MagicMock()
monkeypatch.setattr(engine.input_processor, "assign_request_id", assign_request_id)
engine.output_processor = MagicMock()
engine.engine_core = MagicMock()
request = _engine_core_request(SamplingParams(temperature=0, watermarking=False))
engine.add_request(
request.request_id,
request,
SamplingParams(temperature=0),
)
assert request.sampling_params._watermarking_skipped is False
assign_request_id.assert_called_once_with(request)
def test_direct_async_engine_trace_replay_warns_and_disables(
caplog_vllm, monkeypatch, reset_warning_once
):
engine = object.__new__(AsyncLLM)
engine.engine_core = SimpleNamespace(
resources=SimpleNamespace(engine_dead=False),
shutdown=lambda **kwargs: None,
add_request_async=AsyncMock(),
)
engine.output_handler = None
engine.input_processor = _input_processor()
engine.vllm_config = engine.input_processor.vllm_config
engine.output_processor = MagicMock()
engine.log_requests = False
monkeypatch.setattr(engine, "_run_output_handler", MagicMock())
request = _engine_core_request(SamplingParams(trace_decode_token_ids=[1]))
async def add_request() -> None:
await engine.add_request(
request.request_id,
request,
SamplingParams(watermarking=False),
)
with caplog_vllm.at_level("WARNING"):
asyncio.run(add_request())
assert "trace replay requests will run without watermarking" in caplog_vllm.text
assert request.sampling_params.watermarking is False
assert request.sampling_params._watermarking_skipped is True
def test_direct_async_engine_greedy_request_warns_and_disables_watermarking(
caplog_vllm, monkeypatch, reset_warning_once
):
engine = object.__new__(AsyncLLM)
engine.engine_core = SimpleNamespace(
resources=SimpleNamespace(engine_dead=False),
shutdown=lambda **kwargs: None,
add_request_async=AsyncMock(),
)
engine.output_handler = None
engine.input_processor = _input_processor()
engine.vllm_config = engine.input_processor.vllm_config
engine.output_processor = MagicMock()
engine.log_requests = False
monkeypatch.setattr(engine, "_run_output_handler", MagicMock())
request = _engine_core_request(SamplingParams(temperature=0))
async def add_request() -> None:
await engine.add_request(
request.request_id,
request,
SamplingParams(watermarking=False),
)
with caplog_vllm.at_level("WARNING"):
asyncio.run(add_request())
assert "subsequent greedy requests will use ordinary greedy sampling" in (
caplog_vllm.text
)
assert request.sampling_params.watermarking is False
assert request.sampling_params._watermarking_skipped is True
@pytest.mark.parametrize(
("kwargs", "server_uses_watermarking", "skipped"),
[
({}, True, False),
({"temperature": 0}, True, True),
({"temperature": 0, "watermarking": False}, True, False),
({"temperature": 0}, False, False),
],
)
def test_apply_watermarking_records_skipped_requests(
kwargs, server_uses_watermarking, skipped, reset_warning_once
):
params = SamplingParams(**kwargs)
processor = _input_processor(server_uses_watermarking)
processor.apply_watermarking(params, processor.resolve_watermarking(params))
assert params._watermarking_skipped is skipped
# Resolving an already resolved request keeps the flag.
processor.apply_watermarking(params, processor.resolve_watermarking(params))
assert params._watermarking_skipped is skipped
def test_shared_sampling_params_are_not_rewritten_by_admission(reset_warning_once):
"""A params object reused across prompts/engines must keep inheriting."""
params = SamplingParams(temperature=0.8)
assert _validate(params, server_uses_watermarking=False) is False
assert params.watermarking is None
# Same object submitted to a watermarked engine must still be watermarked.
assert _validate(params, server_uses_watermarking=True) is True
greedy = SamplingParams(temperature=0.0)
assert _validate(greedy) is False
greedy.temperature = 0.8
assert _validate(greedy) is True
def test_shared_beam_search_params_are_not_rewritten_by_admission(
caplog_vllm, reset_warning_once
):
"""Beam search must keep resolving per engine after reuse."""
params = BeamSearchParams(beam_width=2, max_tokens=4, temperature=0.8)
# A plain engine leaves the tri-state unresolved on the caller's object.
assert _input_processor(False).resolve_watermarking(params) is False
assert params.watermarking is None
# The same object on a watermarked engine is warned about and still
# resolves to False, without the first decision being written back.
with caplog_vllm.at_level("WARNING"):
assert _input_processor(True).resolve_watermarking(params) is False
assert "beam search requests will run without watermarking" in caplog_vllm.text
assert params.watermarking is None
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Generator
import pytest
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
from vllm.sampling_params import StructuredOutputsParams
MODEL = "hmellor/tiny-random-LlamaForCausalLM"
MAX_NUM_BATCHED_TOKENS = 32
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.fixture(scope="module")
def watermarked_llm(vllm_runner) -> Generator[LLM, None, None]:
with vllm_runner(
MODEL,
dtype="half",
enforce_eager=True,
enable_prefix_caching=True,
enable_chunked_prefill=True,
max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS,
max_model_len=128,
max_num_seqs=4,
disable_log_stats=False,
watermark_config={"algorithm": "gumbel", "key": 42},
) as runner:
yield runner.llm
def test_watermark_with_combined_sampling_controls(watermarked_llm: LLM):
params = SamplingParams(
temperature=0.8,
top_k=8,
top_p=0.9,
min_p=0.01,
presence_penalty=0.2,
frequency_penalty=0.2,
repetition_penalty=1.1,
logit_bias={10: 100.0},
allowed_token_ids=[10],
max_tokens=8,
ignore_eos=True,
)
output = watermarked_llm.generate("Sampling smoke test", params)[0]
assert list(output.outputs[0].token_ids) == [10] * 8
# The assertion above holds with or without watermarking, because
# allowed_token_ids leaves a single candidate. Repeat with a wider allowed
# set, where only watermarked sampling is reproducible without a seed.
wide = SamplingParams(
temperature=0.8,
top_k=8,
top_p=0.9,
min_p=0.01,
presence_penalty=0.2,
frequency_penalty=0.2,
repetition_penalty=1.1,
allowed_token_ids=list(range(10, 210)),
max_tokens=8,
ignore_eos=True,
)
first = watermarked_llm.generate("Sampling smoke test", wide)[0]
second = watermarked_llm.generate("Sampling smoke test", wide)[0]
assert set(first.outputs[0].token_ids) <= set(range(10, 210))
assert first.outputs[0].token_ids == second.outputs[0].token_ids
def test_watermark_with_penalties_and_nucleus_sampling(watermarked_llm: LLM):
params = SamplingParams(
temperature=0.8,
top_k=8,
top_p=0.9,
min_p=0.01,
presence_penalty=0.2,
frequency_penalty=0.2,
repetition_penalty=1.1,
max_tokens=8,
ignore_eos=True,
)
first = watermarked_llm.generate("Penalty smoke test", params)[0]
second = watermarked_llm.generate("Penalty smoke test", params)[0]
assert len(first.outputs[0].token_ids) == 8
assert first.outputs[0].token_ids == second.outputs[0].token_ids
def test_plain_gumbel_seeded_parallel_sampling_is_deterministic(
watermarked_llm: LLM,
):
output = watermarked_llm.generate(
"Seeded parallel sampling smoke test",
SamplingParams(
n=2,
seed=42,
temperature=0.8,
max_tokens=8,
ignore_eos=True,
),
)[0]
assert len(output.outputs) == 2
assert all(len(candidate.token_ids) == 8 for candidate in output.outputs)
assert output.outputs[0].token_ids == output.outputs[1].token_ids
# Plain Gumbel keys on the token context, so the request seed does not
# change a watermarked draw. Pin that down rather than implying the seed
# is what makes the candidates match.
unseeded = watermarked_llm.generate(
"Seeded parallel sampling smoke test",
SamplingParams(
n=2,
temperature=0.8,
max_tokens=8,
ignore_eos=True,
),
)[0]
assert unseeded.outputs[0].token_ids == output.outputs[0].token_ids
@pytest.mark.parametrize(
"structured_outputs",
[
StructuredOutputsParams(choice=["A", "B"]),
StructuredOutputsParams(grammar='root ::= "A" | "B"'),
],
ids=["choice", "grammar"],
)
def test_watermark_with_structured_output(
watermarked_llm: LLM, structured_outputs: StructuredOutputsParams
):
params = SamplingParams(
temperature=0.8,
max_tokens=8,
structured_outputs=structured_outputs,
)
output = watermarked_llm.generate("Choose A or B:", params)[0]
assert output.outputs[0].text.strip() in ("A", "B")
def test_watermark_with_wide_grammar_is_deterministic(watermarked_llm: LLM):
# A wider grammar makes deterministic watermarking observable.
params = SamplingParams(
temperature=0.8,
max_tokens=8,
structured_outputs=StructuredOutputsParams(
grammar="root ::= [A-Z] [A-Z] [A-Z] [A-Z] [A-Z] [A-Z]"
),
)
first = watermarked_llm.generate("Emit six letters:", params)[0]
second = watermarked_llm.generate("Emit six letters:", params)[0]
assert first.outputs[0].text.strip().isupper()
assert first.outputs[0].token_ids == second.outputs[0].token_ids
def test_watermark_with_prefix_cache_and_chunked_prefill(watermarked_llm: LLM):
prompt = {"prompt_token_ids": [101] * 48}
params = SamplingParams(temperature=0.8, max_tokens=8, ignore_eos=True)
assert len(prompt["prompt_token_ids"]) > MAX_NUM_BATCHED_TOKENS
cold = watermarked_llm.generate(prompt, params)[0]
cached = watermarked_llm.generate(prompt, params)[0]
assert cached.num_cached_tokens > 0
assert cold.outputs[0].token_ids == cached.outputs[0].token_ids
def test_mixed_watermarked_batch(watermarked_llm: LLM):
prompts = [
{"prompt_token_ids": [101] * 32 + [102]},
{"prompt_token_ids": [201] * 32 + [202]},
]
marked = SamplingParams(temperature=0.8, max_tokens=8, ignore_eos=True)
ordinary = SamplingParams(
temperature=0.8,
seed=123,
max_tokens=8,
ignore_eos=True,
watermarking=False,
)
batched = watermarked_llm.generate(prompts, [marked, ordinary])
marked_alone = watermarked_llm.generate(prompts[0], marked)[0]
ordinary_alone = watermarked_llm.generate(prompts[1], ordinary)[0]
assert batched[0].outputs[0].token_ids == marked_alone.outputs[0].token_ids
assert batched[1].outputs[0].token_ids == ordinary_alone.outputs[0].token_ids
-284
View File
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.config.watermarking import WatermarkConfig
from vllm.platforms import current_platform
from vllm.v1.watermarking import (
DualKeyGumbelWatermarkDetector,
DualKeyGumbelWatermarker,
GumbelWatermarkDetector,
GumbelWatermarker,
derive_watermark_key,
)
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
from vllm.v1.watermarking.watermarker import RandomSampler
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
def test_gamma_survival_integer_shape():
assert _gamma_survival_integer_shape(0.0, 4) == 1.0
assert _gamma_survival_integer_shape(2.0, 1) == pytest.approx(0.1353352832)
def test_detector_deduplicates_repeated_prf_inputs_by_default():
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 1, 1, 1, 1])
assert detection.num_scored_tokens == 2
def test_detector_can_score_repeated_prf_inputs():
detection = GumbelWatermarkDetector(
key=42, context_width=1, deduplicate_contexts=False
).detect([1, 1, 1, 1, 1])
assert detection.num_scored_tokens == 5
def test_detector_deduplicates_context_even_when_target_differs():
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 2, 1, 3])
assert detection.num_scored_tokens == 3
def test_watermarker_respects_filtered_token_support():
logits = torch.full((2, 8), -torch.inf)
logits[0, 3] = 0
logits[1, 6] = 0
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
token_ids = (
GumbelWatermarker(key=42)
.sample(logits, contexts, lambda values: None)
.token_ids
)
assert torch.equal(token_ids, torch.tensor([3, 6]))
def test_dual_key_detector_scores_each_token_against_both_keys():
token_ids = [1, 2, 3, 4, 5]
key_a_detector = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_a"),
context_width=1,
deduplicate_contexts=False,
)
key_b_detector = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_b"),
context_width=1,
deduplicate_contexts=False,
)
detector = DualKeyGumbelWatermarkDetector(
key=42,
context_width=1,
deduplicate_contexts=False,
alpha=0.5,
)
key_a = key_a_detector.detect(token_ids)
key_b = key_b_detector.detect(token_ids)
dual = detector.detect(token_ids)
assert dual.score == pytest.approx((key_a.score + key_b.score) / 2)
assert dual.p_value == pytest.approx(
_gamma_survival_integer_shape(dual.score * 2, dual.num_scored_tokens * 2)
)
@pytest.mark.parametrize("alpha", [0.2, 0.7])
def test_dual_key_detector_weights_and_recalibrates_scores(alpha: float):
token_ids = [1, 2, 3, 4, 5]
key_a = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_a"), deduplicate_contexts=False
).detect(token_ids)
key_b = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_b"), deduplicate_contexts=False
).detect(token_ids)
dual = DualKeyGumbelWatermarkDetector(
key=42, deduplicate_contexts=False, alpha=alpha
).detect(token_ids)
variance = (1 - alpha) ** 2 + alpha**2
expected_p_value = torch.special.gammaincc(
torch.tensor(dual.num_scored_tokens / variance, dtype=torch.float64),
torch.tensor(dual.score / variance, dtype=torch.float64),
).item()
assert dual.score == pytest.approx((1 - alpha) * key_a.score + alpha * key_b.score)
assert dual.p_value == pytest.approx(expected_p_value)
def test_dual_key_detector_rejects_invalid_alpha():
with pytest.raises(ValueError, match="alpha must be between 0 and 1"):
DualKeyGumbelWatermarkDetector(key=42, alpha=1.1)
def test_dual_key_alpha_defaults_match():
config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
watermarker = DualKeyGumbelWatermarker(key=42)
detector = DualKeyGumbelWatermarkDetector(key=42)
assert detector.alpha == watermarker.alpha == config.alpha
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("key", [1, 2**32, 42, 15726070495360670683])
@pytest.mark.parametrize("context_width", [1, 4, 16])
def test_fused_watermarker_matches_cpu(key: int, context_width: int):
torch.manual_seed(0)
contexts = torch.randint(0, 248320, (32, context_width), dtype=torch.int64)
logits = torch.randn(32, 8193)
watermarker = GumbelWatermarker(key, context_width)
expected = watermarker.sample(logits, contexts).token_ids
actual = watermarker.sample(logits.cuda(), contexts.cuda()).token_ids.cpu()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_fused_watermarker_handles_nan_logits():
contexts = torch.zeros((2, 4), dtype=torch.int64, device="cuda")
logits = torch.full((2, 1025), float("nan"), device="cuda")
watermarker = GumbelWatermarker(key=42, context_width=4)
token_ids = watermarker.sample(logits, contexts).token_ids
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_fused_watermarker_handles_noncontiguous_inputs():
contexts = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")[
:, ::2
]
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
watermarker = GumbelWatermarker(key=42, context_width=4)
expected = watermarker.sample(logits.contiguous(), contexts.contiguous()).token_ids
actual = watermarker.sample(logits, contexts).token_ids
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("use_fp64", [False, True])
def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool):
torch.manual_seed(0)
logits = torch.randn(8, 8193, device="cuda")
context_storage = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")
contexts = context_storage[:, ::2]
repeated_mask = torch.tensor(
[False, False, False, True, False, False, True, False], device="cuda"
)
watermarking = torch.tensor(
[True, False, True, True, False, True, True, False], device="cuda"
)
req_indices = torch.tensor([3, 1, 7, 0, 4, 2, 6, 5], device="cuda")
temperatures = torch.ones(8, dtype=torch.float32, device="cuda")
temperatures[2] = 0
seeds = torch.arange(8, dtype=torch.int64, device="cuda") + 1000
positions = torch.arange(8, dtype=torch.int64, device="cuda") + 100
watermark_mask = (
watermarking[req_indices] & (temperatures[req_indices] != 0) & ~repeated_mask
)
watermarked = philox_gumbel_sample(logits, contexts, 42)
ordinary = gumbel_sample(
logits,
req_indices,
temperatures,
seeds,
positions,
apply_temperature=False,
is_drafting=False,
use_fp64=use_fp64,
)
expected = torch.where(watermark_mask, watermarked, ordinary)
actual = philox_gumbel_sample(
logits,
contexts,
42,
skip_mask=~watermark_mask,
expanded_idx_mapping=req_indices,
temperatures=temperatures,
seeds=seeds,
positions=positions,
use_fp64=use_fp64,
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
torch.manual_seed(0)
num_rows, vocab_size, num_steps = 5, 4099, 3
logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16)
temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda")
processed_logits = logits / torch.where(
temperatures == 0, 1, temperatures
).unsqueeze(-1)
contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda")
idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda")
seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10
positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20
# The skipped rows must divide by a temperature other than 1, or the
# unprocessed logits the cache stores are the logits the sampler sees.
# Row 0 keeps a greedy row in the skipped set.
skip_mask = torch.tensor([True, True, False, False, True], device="cuda")
cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps
cache = torch.zeros(
num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda"
)
ordinary = gumbel_sample(
logits,
idx_mapping,
temperatures,
seeds,
positions,
apply_temperature=True,
is_drafting=True,
logits_cache=cache,
logits_cache_col=cols,
)
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
expected = torch.where(skip_mask, ordinary, watermarked)
actual = (
GumbelWatermarker(key=42, context_width=4)
.sample(
processed_logits,
contexts,
random_sampler=RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperatures,
seeds=seeds,
positions=positions,
is_drafting=True,
logits_cache=cache,
logits_cache_col=cols,
logits_cache_source=logits,
),
skip_mask=skip_mask,
)
.token_ids
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size]
assert torch.equal(stored.view(torch.int16), logits.view(torch.int16))
assert not cache[:, :, -1].any()
@@ -1,106 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import pytest
from tests.utils import multi_gpu_test
from vllm import SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.v1.engine.async_llm import AsyncLLM
MODEL = "hmellor/tiny-random-LlamaForCausalLM"
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"parallel_kwargs",
[
{"tensor_parallel_size": 2, "enable_batch_sharded_sampling": True},
{"pipeline_parallel_size": 2},
],
ids=["tp2-sharded-sampling", "pp2"],
)
def test_watermarked_generation_with_model_parallelism(vllm_runner, parallel_kwargs):
prompts = ["Parallel watermark smoke"] * 2
with vllm_runner(
MODEL,
dtype="half",
enforce_eager=True,
max_model_len=128,
max_num_seqs=4,
watermark_config={"algorithm": "gumbel", "key": 42},
**parallel_kwargs,
) as runner:
first = runner.llm.generate(
prompts,
SamplingParams(
temperature=0.8,
seed=1,
max_tokens=8,
ignore_eos=True,
),
)
second = runner.llm.generate(
prompts,
SamplingParams(
temperature=0.8,
seed=2,
max_tokens=8,
ignore_eos=True,
),
)
first_tokens = [output.outputs[0].token_ids for output in first]
second_tokens = [output.outputs[0].token_ids for output in second]
assert all(len(token_ids) == 8 for token_ids in first_tokens)
assert first_tokens[0] == first_tokens[1]
assert first_tokens == second_tokens
@multi_gpu_test(num_gpus=2)
def test_watermarked_generation_with_data_parallelism():
async def run() -> None:
engine = AsyncLLM.from_engine_args(
AsyncEngineArgs(
model=MODEL,
dtype="half",
data_parallel_size=2,
enforce_eager=True,
max_model_len=128,
max_num_seqs=4,
watermark_config={"algorithm": "gumbel", "key": 42},
)
)
async def generate(index: int) -> tuple[int, ...]:
final_output = None
async for output in engine.generate(
"Data-parallel watermark smoke",
SamplingParams(
temperature=0.8,
seed=index,
max_tokens=8,
ignore_eos=True,
),
request_id=f"watermark-dp-{index}",
data_parallel_rank=index,
):
final_output = output
assert final_output is not None
return tuple(final_output.outputs[0].token_ids)
try:
token_ids = await asyncio.gather(generate(0), generate(1))
assert len(token_ids[0]) == 8
assert token_ids[0] == token_ids[1]
finally:
engine.shutdown()
try:
asyncio.run(run())
finally:
cleanup_dist_env_and_memory()
-44
View File
@@ -1,44 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.watermarking.prfs import PhiloxPRF
def test_philox_compatibility_vector():
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
token_ids = torch.tensor([7, 8])
expected_mantissas = [[1661833, 10571167], [9742331, 13375724]]
expected = (
(torch.tensor(expected_mantissas, dtype=torch.float64) + 1) / (2**24 + 1)
).to(torch.float32)
assert torch.equal(PhiloxPRF(42).uniform(contexts, token_ids), expected)
def test_prf_pairs_contexts_with_target_tokens():
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
token_ids = torch.tensor([[7], [8]])
assert PhiloxPRF(42).uniform(contexts, token_ids).shape == (2, 1)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("key", [42, 15726070495360670683])
@pytest.mark.parametrize("context_width", [1, 4, 16])
def test_philox_accelerator_matches_cpu(key: int, context_width: int):
contexts = torch.arange(2 * context_width, dtype=torch.int64).reshape(
2, context_width
)
token_ids = torch.arange(1024, dtype=torch.int64)
prf = PhiloxPRF(key)
expected = prf.uniform(contexts, token_ids)
actual = prf.uniform(contexts.cuda(), token_ids.cuda()).cpu()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
File diff suppressed because it is too large Load Diff
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
from vllm.v1.metrics.reader import Counter
from vllm.v1.watermarking import GumbelWatermarkDetector
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_llm_generated_sequence_is_watermarked(vllm_runner):
"""Engine-level watermarking must produce tokens the detector recognizes."""
watermark_config = {"key": 42, "context_width": 4}
detector = GumbelWatermarkDetector(**watermark_config)
runner = vllm_runner(
"facebook/opt-125m",
watermark_config=watermark_config,
seed=0,
enforce_eager=True,
max_model_len=512,
gpu_memory_utilization=0.2,
disable_log_stats=False,
)
request = "Tell me a story about an explorer who discovers a mysterious island."
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
params_greedy = SamplingParams(temperature=0.0, max_tokens=16)
with runner:
output_use_wm = runner.llm.generate(request, params_use_wm)
output_no_wm = runner.llm.generate(request, params_no_wm)
runner.llm.generate(request, params_greedy)
metrics = runner.llm.get_metrics()
# The greedy request is skipped; the opt-out one is not watermarked.
counters = [m for m in metrics if isinstance(m, Counter)]
status = {
m.labels["status"]: m.value
for m in counters
if m.name == "vllm:watermark_requests"
}
assert status == {"watermarked": 1, "skipped": 1, "not_watermarked": 1}
num_success = sum(m.value for m in counters if m.name == "vllm:request_success")
assert sum(status.values()) == num_success
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
assert result_use_wm.is_watermarked, result_use_wm
recorded_p_value = 9.83e-20
assert result_use_wm.p_value < recorded_p_value * 10 # tolerance
assert not result_no_wm.is_watermarked, result_no_wm