Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -25,7 +25,7 @@ def make_manager(
|
||||
num_reqs, num_steps = confidences.shape
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.num_speculative_steps = num_steps
|
||||
manager._stale_confidences = [SimpleNamespace(np=confidences)]
|
||||
manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer.
|
||||
manager._stale_idx = 0
|
||||
manager.req_states = SimpleNamespace(
|
||||
req_id_to_index={"low": 0, "high": 1},
|
||||
@@ -176,6 +176,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
|
||||
|
||||
assert draft_budget == 1
|
||||
@@ -183,14 +184,16 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
assert num_non_draft_tokens == {"low": 1, "high": 1}
|
||||
|
||||
|
||||
def test_profiled_batches_seed_cost_curves_via_consumer():
|
||||
def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch):
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
|
||||
manager.num_speculative_steps = 7
|
||||
manager.num_bonus_tokens = 1
|
||||
curves: dict[str, list[tuple[int, float]]] = {}
|
||||
manager.set_cost_curves = lambda draft, verify: curves.update(
|
||||
draft=draft, verify=verify
|
||||
monkeypatch.setattr(
|
||||
manager,
|
||||
"set_cost_curves",
|
||||
lambda draft, verify: curves.update(draft=draft, verify=verify),
|
||||
)
|
||||
|
||||
timings = [
|
||||
@@ -258,6 +261,7 @@ def test_budget_caps_at_one_rejection_sampler_chunk():
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget <= 1
|
||||
|
||||
@@ -285,6 +289,7 @@ def test_zero_budget_rebuilds_cpu_cu_num_logits():
|
||||
{"low": 3, "high": 3, "prefill": 40},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget == 0
|
||||
|
||||
@@ -323,6 +328,7 @@ def test_zero_budget_keeps_one_grammar_row_per_scheduled_draft():
|
||||
manager.get_num_tokens(
|
||||
{"low": 3, "high": 3, "prefill": 40}, scheduled_spec_decode_tokens
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
assert manager._batch_budget[2] == 0
|
||||
|
||||
req_ids = ["low", "high", "prefill"]
|
||||
|
||||
@@ -434,7 +434,9 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch):
|
||||
for projection in projections
|
||||
]
|
||||
qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers(
|
||||
model, layers_attn, has_bias=False
|
||||
model,
|
||||
layers_attn,
|
||||
has_bias=False,
|
||||
)
|
||||
assert model._fused_kv_weight is None
|
||||
assert all(not hasattr(projection, "weight") for projection in projections)
|
||||
|
||||
@@ -143,7 +143,7 @@ def test_dflash_drafter_window_reserves_bonus_token():
|
||||
# window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101)
|
||||
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96))
|
||||
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97))
|
||||
assert not input_fits_in_drafter(dflash_runner, None) # no metadata
|
||||
assert not input_fits_in_drafter(dflash_runner, None)
|
||||
|
||||
# Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100).
|
||||
plain_runner = SimpleNamespace(
|
||||
|
||||
@@ -56,6 +56,7 @@ def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
|
||||
):
|
||||
shared_head = SharedHead(config=config, prefix="shared_head")
|
||||
|
||||
assert shared_head.head is not None
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
with torch.no_grad():
|
||||
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
|
||||
|
||||
@@ -25,7 +25,7 @@ def _create_vllm_config_for_dsd(
|
||||
max_num_seqs: int,
|
||||
max_spec_tokens: int,
|
||||
*,
|
||||
cudagraph_mode: str = "FULL_AND_PIECEWISE",
|
||||
cudagraph_mode: CUDAGraphMode = CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd: bool = True,
|
||||
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
@@ -199,7 +199,7 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=512,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=True,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
@@ -316,7 +316,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=False,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
@@ -382,7 +382,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=True,
|
||||
num_spec_per_batch_size=num_spec_per_batch_size,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Literal
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
@@ -31,6 +32,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
|
||||
@@ -46,13 +48,13 @@ DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def _create_proposer(
|
||||
method: str,
|
||||
method: Literal["eagle", "eagle3", "draft_model", "dflash"],
|
||||
num_speculative_tokens: int,
|
||||
attention_backend: str | None = None,
|
||||
parallel_drafting: bool = False,
|
||||
rejection_sample_method: str = "standard",
|
||||
draft_sample_method: str = "greedy",
|
||||
) -> EagleProposer:
|
||||
rejection_sample_method: Literal["standard", "synthetic", "block"] = "standard",
|
||||
draft_sample_method: Literal["greedy", "probabilistic"] = "greedy",
|
||||
) -> SpecDecodeBaseProposer:
|
||||
# Method-dependent setup
|
||||
if method == "eagle":
|
||||
target_model_dir = model_dir
|
||||
@@ -102,9 +104,14 @@ def _create_proposer(
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
),
|
||||
attention_config=AttentionConfig(backend=attention_backend),
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum[attention_backend]
|
||||
if attention_backend
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
proposer: SpecDecodeBaseProposer
|
||||
if method == "dflash":
|
||||
proposer = DFlashProposer(vllm_config=vllm_config, device=device)
|
||||
elif "eagle" in method:
|
||||
@@ -135,7 +142,7 @@ def test_prepare_next_token_ids():
|
||||
)
|
||||
|
||||
mock_num_scheduled_tokens = {req_id: 0 for req_id in req_ids}
|
||||
mock_requests = {}
|
||||
mock_requests: dict[str, CachedRequestState] = {}
|
||||
for req_id in req_ids:
|
||||
mock_request = mock.MagicMock(spec=CachedRequestState)
|
||||
# Each request will have a backup next token id of 10, 20, 30, 40
|
||||
@@ -948,13 +955,12 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer._draft_attn_layer_names,
|
||||
layer_names=sorted(proposer._draft_attn_layer_names),
|
||||
vllm_config=proposer.vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Mock runner and draft_attn_groups for attention metadata building
|
||||
proposer.runner = mock.MagicMock()
|
||||
# Mock draft_attn_groups for attention metadata building.
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
@@ -1054,11 +1060,10 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
|
||||
)
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer._draft_attn_layer_names,
|
||||
layer_names=sorted(proposer._draft_attn_layer_names),
|
||||
vllm_config=proposer.vllm_config,
|
||||
device=device,
|
||||
)
|
||||
proposer.runner = mock.MagicMock()
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
@@ -1133,6 +1138,7 @@ def test_set_inputs_first_pass_dflash():
|
||||
|
||||
num_speculative_tokens = 3
|
||||
proposer = _create_proposer("dflash", num_speculative_tokens)
|
||||
assert isinstance(proposer, DFlashProposer)
|
||||
mask_token_id = proposer.parallel_drafting_token_id
|
||||
|
||||
# Setup batch with 3 requests
|
||||
|
||||
@@ -93,7 +93,7 @@ def _run_build(
|
||||
|
||||
with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata):
|
||||
EagleSpeculator._build_uniform_attn_metadata(
|
||||
fake, # type: ignore[arg-type]
|
||||
fake,
|
||||
batch_desc=batch_desc,
|
||||
num_reqs=num_reqs,
|
||||
num_query_per_req=num_query_per_req,
|
||||
|
||||
@@ -146,7 +146,7 @@ def test_prepare_next_token_ids_padded():
|
||||
mock_input_batch.vocab_size = 100
|
||||
mock_input_batch.num_tokens_no_spec = np.array([5] * num_requests)
|
||||
|
||||
mock_requests = {}
|
||||
mock_requests: dict[str, CachedRequestState] = {}
|
||||
for req_id in req_ids:
|
||||
mock_request = mock.MagicMock(spec=CachedRequestState)
|
||||
# Each request will have a backup next token id of 10, 20, 30, 40
|
||||
|
||||
@@ -11,6 +11,7 @@ from torch import nn
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
DeviceConfig,
|
||||
LoadConfig,
|
||||
VllmConfig,
|
||||
@@ -122,7 +123,7 @@ def test_lilicorr_matches_exported_head(head_width, slots, dtype):
|
||||
set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cuda"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
),
|
||||
):
|
||||
@@ -183,7 +184,7 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch):
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
for slots in (1, 3):
|
||||
@@ -268,7 +269,11 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz
|
||||
)
|
||||
monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1))
|
||||
ids, values = logits_processor.LogitsProcessor.get_top_k_tokens(
|
||||
fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True
|
||||
fake,
|
||||
lm_head,
|
||||
torch.empty(2, 1),
|
||||
2,
|
||||
return_log_probs=True,
|
||||
)
|
||||
expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2)
|
||||
torch.testing.assert_close(ids, expected.indices)
|
||||
@@ -349,22 +354,35 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
|
||||
compute_candidates=lambda h: (candidates, log_probs),
|
||||
model=SimpleNamespace(lilicorr=score, candidate_selector=score),
|
||||
)
|
||||
spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4)
|
||||
if lilicorr:
|
||||
assert isinstance(spec, LiLiCorrSpeculator)
|
||||
monkeypatch.setattr(
|
||||
spec,
|
||||
"target_embeddings",
|
||||
lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4),
|
||||
raising=False,
|
||||
)
|
||||
spec.anchor_hidden.copy_(torch.arange(12).view(3, 4))
|
||||
spec.anchor_valid[:2] = True
|
||||
spec.candidate_sampler.sample = sample
|
||||
spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args)
|
||||
monkeypatch.setattr(spec.candidate_sampler, "sample", sample)
|
||||
monkeypatch.setattr(
|
||||
spec, "_maybe_predict_acceptance", lambda *args: adaptive_inputs.extend(args)
|
||||
)
|
||||
spec._generate_draft(3, 9, None, None, None)
|
||||
|
||||
expected_ids = candidates.view(3, 2, 4)
|
||||
expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids
|
||||
expected_first = (
|
||||
spec.target_embeddings(expected_ids)
|
||||
if isinstance(spec, LiLiCorrSpeculator)
|
||||
else expected_ids
|
||||
)
|
||||
torch.testing.assert_close(scoring_inputs[0], expected_first)
|
||||
torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4))
|
||||
torch.testing.assert_close(
|
||||
scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4)
|
||||
)
|
||||
if lilicorr:
|
||||
assert isinstance(spec, LiLiCorrSpeculator)
|
||||
torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden)
|
||||
torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid)
|
||||
else:
|
||||
@@ -472,7 +490,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
|
||||
):
|
||||
from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM
|
||||
|
||||
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM)
|
||||
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) # type: ignore[type-abstract]
|
||||
nn.Module.__init__(wrapper)
|
||||
wrapper.model = LiLiCorr.__new__(LiLiCorr)
|
||||
nn.Module.__init__(wrapper.model)
|
||||
@@ -480,7 +498,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
wrapper.model.lilicorr = LiLiCorrHead(
|
||||
@@ -615,7 +633,7 @@ def owned_head_model(monkeypatch):
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
model = LiLiCorrForCausalLM(vllm_config=vllm_config)
|
||||
@@ -720,7 +738,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
head = LiLiCorrHead(
|
||||
|
||||
@@ -16,6 +16,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
import vllm.v1.spec_decode.llm_base_proposer as llm_base_proposer
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
|
||||
SCHEDULER_BLOCK_SIZE = 256
|
||||
@@ -61,10 +62,10 @@ def _make_proposer(
|
||||
return proposer
|
||||
|
||||
|
||||
def _make_kv_cache_config(layer_names: set[str]) -> SimpleNamespace:
|
||||
def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
|
||||
spec = SimpleNamespace(block_size=SCHEDULER_BLOCK_SIZE)
|
||||
group = SimpleNamespace(layer_names=list(layer_names), kv_cache_spec=spec)
|
||||
return SimpleNamespace(kv_cache_groups=[group])
|
||||
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -81,7 +82,9 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
|
||||
# The metadata builder keeps receiving the kernel block size as well.
|
||||
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
group = proposer.draft_attn_groups[0]
|
||||
assert isinstance(group, _FakeAttentionGroup)
|
||||
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
|
||||
|
||||
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config):
|
||||
index_kpool=4,
|
||||
)
|
||||
vllm_config = mock.MagicMock()
|
||||
vllm_config.speculative_config.draft_model_config.hf_config = config
|
||||
vllm_config.speculative_config.draft_model_config.hf_text_config = config
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 4
|
||||
|
||||
with (
|
||||
@@ -220,7 +220,6 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
|
||||
device=device,
|
||||
)
|
||||
|
||||
proposer.runner = mock.MagicMock()
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backend and engine-reasoner checks that are not manager-flow tests."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -17,7 +19,9 @@ TOKENIZER = "gpt2"
|
||||
NUM_SPEC_TOKENS = 4
|
||||
|
||||
|
||||
def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
def _make_manager_and_request(
|
||||
backend: Literal["xgrammar", "guidance"], prompt_str: str = '{"a": "b"}'
|
||||
):
|
||||
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
|
||||
prompt = tokenizer.encode(prompt_str)
|
||||
|
||||
@@ -33,6 +37,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
sampling_params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
|
||||
)
|
||||
assert sampling_params.structured_outputs is not None
|
||||
sampling_params.structured_outputs._backend = backend
|
||||
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
|
||||
|
||||
@@ -43,6 +48,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
pooling_params=None,
|
||||
)
|
||||
manager.grammar_init(request)
|
||||
assert request.structured_output_request is not None
|
||||
while not request.structured_output_request._check_grammar_completion():
|
||||
continue
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ def test_watermarked_bonus_uses_target_key():
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[1:], lambda _: None
|
||||
target_logits.unsqueeze(0), contexts[1:]
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 2
|
||||
@@ -196,13 +196,189 @@ def test_watermarked_recovery_uses_target_key():
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[:1], lambda _: None
|
||||
target_logits.unsqueeze(0), contexts[:1]
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
|
||||
|
||||
def test_repeated_draft_context_does_not_change_acceptance():
|
||||
torch.manual_seed(0)
|
||||
num_trials = 8
|
||||
# A draft distribution of its own, so the drafts are not accepted outright.
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
draft_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat(
|
||||
num_trials, 1
|
||||
),
|
||||
watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat(
|
||||
num_trials
|
||||
),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
# Some drafts must actually be rejected, or the ordinary-residual branch
|
||||
# the skip mask selects is never reached.
|
||||
assert (expected_num_sampled == 1).any()
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_context_skip_mask_is_read_per_row_in_a_block():
|
||||
"""A block of three drafts, accepted at row 0 and rejected at row 1.
|
||||
|
||||
The skip mask selects the sampler per row, so the row the target recovers
|
||||
at decides which key draws the recovery token.
|
||||
"""
|
||||
num_speculative_steps = 3
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
|
||||
draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
# Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is
|
||||
# accepted; row 1 drafts a token the target gives no mass, so it is not.
|
||||
inputs["draft_sampled"][1] = 3
|
||||
inputs["draft_sampled"][2] = 0
|
||||
contexts = torch.tensor(
|
||||
[[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda"
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(
|
||||
**inputs, num_speculative_steps=num_speculative_steps
|
||||
)
|
||||
assert expected_num_sampled.item() == 2
|
||||
assert expected[0, 0].item() == 3
|
||||
|
||||
watermarker = GumbelWatermarker(key=42, context_width=2)
|
||||
skipped, skipped_num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
|
||||
assert skipped_num_sampled.item() == 2
|
||||
torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0)
|
||||
|
||||
watermarked, watermarked_num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.zeros(
|
||||
num_speculative_steps + 1, dtype=torch.bool, device="cuda"
|
||||
),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
# The residual equals the target to within e**-60, so the recovery token is
|
||||
# the target key's draw on the rejected row's context.
|
||||
recovery = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[1:2], lambda _: None
|
||||
)
|
||||
|
||||
assert watermarked_num_sampled.item() == 2
|
||||
assert watermarked[0, 0].item() == 3
|
||||
assert watermarked[0, 1] == recovery.token_ids[0]
|
||||
|
||||
|
||||
def test_repeated_recovery_context_uses_ordinary_residual_sample():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
inputs["draft_sampled"][1] = 0
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"),
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([True, False], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_bonus_context_uses_ordinary_target_sample():
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
target_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"),
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([False, True], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows():
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
target_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=2,
|
||||
)
|
||||
inputs["temperature"][1] = 0
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"),
|
||||
watermarking=torch.tensor([False, True], device="cuda"),
|
||||
watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
for row, length in enumerate(num_sampled):
|
||||
torch.testing.assert_close(
|
||||
sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
@@ -225,9 +401,7 @@ def test_watermarked_recovery_supports_smaller_draft_vocabulary():
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
expected = watermarker.sample(
|
||||
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
|
||||
)
|
||||
expected = watermarker.sample(target_logits[:4].unsqueeze(0), contexts[:1])
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
@@ -735,6 +909,7 @@ def test_verify_rejects_unproposed_drafts():
|
||||
inputs["expanded_idx_mapping"],
|
||||
inputs["expanded_local_pos"],
|
||||
None,
|
||||
draft_sampled,
|
||||
)
|
||||
return sampled, num_sampled
|
||||
|
||||
|
||||
Reference in New Issue
Block a user