From 3c42385d7dcce60cc231bcd9127b56379f43f823 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:29:56 -0400 Subject: [PATCH] [Refactor] Remove dead or duplicate tests (#58446) Signed-off-by: yewentao256 --- .../pooling/scoring/test_bi_encoder_online.py | 29 ---- .../scoring/test_cross_encoder_online.py | 29 ---- .../attention/test_attention_selector.py | 55 ------- .../attention/test_rocm_attention_selector.py | 73 --------- tests/quantization/test_auto_round.py | 13 -- tests/quantization/test_gptq_v2.py | 106 ------------- tests/v1/core/test_async_scheduler.py | 36 ----- tests/v1/core/test_kv_sharing.py | 32 ---- .../offloading_connector/test_scheduler.py | 53 ------- tests/v1/sample/test_topk_topp_sampler.py | 73 --------- .../test_backup_token_async_spec.py | 147 ------------------ 11 files changed, 646 deletions(-) delete mode 100644 tests/kernels/attention/test_rocm_attention_selector.py delete mode 100644 tests/quantization/test_gptq_v2.py delete mode 100644 tests/v1/spec_decode/test_backup_token_async_spec.py diff --git a/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py b/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py index dac7b6db6c..16392fd338 100644 --- a/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py +++ b/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py @@ -126,35 +126,6 @@ async def test_score_api_queries_str_n_documents_str_n( assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) -@pytest.mark.asyncio -async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer): - text_pairs = [ - [TEXTS_1[0], TEXTS_2[0]], - [TEXTS_1[1], TEXTS_2[1]], - ] - - score_response = requests.post( - server.url_for("score"), - json={ - "model": MODEL_NAME, - "queries": TEXTS_1, - "documents": TEXTS_2, - }, - ) - score_response.raise_for_status() - score = ScoreResponse.model_validate(score_response.json()) - - assert score.id is not None - assert score.data is not None - assert len(score.data) == 2 - - vllm_outputs = [d.score for d in score.data] - hf_outputs = hf_model.predict(text_pairs).tolist() - - for i in range(len(vllm_outputs)): - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) - - @pytest.mark.asyncio async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer): text_pairs = [ diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py index 401bf4ff22..9228f96fe7 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py @@ -143,35 +143,6 @@ async def test_score_api_queries_str_n_documents_str_n( assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) -@pytest.mark.asyncio -async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer): - text_pairs = [ - [TEXTS_1[0], TEXTS_2[0]], - [TEXTS_1[1], TEXTS_2[1]], - ] - - score_response = requests.post( - server.url_for("score"), - json={ - "model": MODEL_NAME, - "queries": TEXTS_1, - "documents": TEXTS_2, - }, - ) - score_response.raise_for_status() - score = ScoreResponse.model_validate(score_response.json()) - - assert score.id is not None - assert score.data is not None - assert len(score.data) == 2 - - vllm_outputs = [d.score for d in score.data] - hf_outputs = hf_model.predict(text_pairs).tolist() - - for i in range(len(vllm_outputs)): - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) - - @pytest.mark.asyncio async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer): text_pairs = [ diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index c5cd4af08f..cb72c27904 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -341,61 +341,6 @@ def test_fp32_fallback(device: str): get_attn_backend(16, torch.float32, None) -def test_flash_attn(monkeypatch: pytest.MonkeyPatch): - """Test FlashAttn validation.""" - pytest.skip( - "Skipping as current backend selector does not " - "handle fallbacks when a backend is explicitly set." - ) - - attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN) - cache_config = CacheConfig(block_size=16) - vllm_config = VllmConfig( - attention_config=attention_config, cache_config=cache_config - ) - - with set_current_vllm_config(vllm_config): - # Unsupported CUDA arch - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _=None: (7, 5)) - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # Reset the monkeypatch for subsequent tests - monkeypatch.undo() - - # Unsupported data type - backend = get_attn_backend(16, torch.float8_e4m3fn, None) - assert backend.get_name() != "FLASH_ATTN" - - # Unsupported kv cache data type - backend = get_attn_backend(16, torch.float16, "fp8") - assert backend.get_name() != "FLASH_ATTN" - - # Unsupported block size - vllm_config.cache_config.block_size = 8 - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # flash-attn is not installed - import sys - - vllm_config.cache_config.block_size = 16 - original_module = sys.modules.get("vllm_flash_attn") - monkeypatch.setitem(sys.modules, "vllm_flash_attn", None) - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # Restore the original module if it existed - if original_module is not None: - monkeypatch.setitem(sys.modules, "vllm_flash_attn", original_module) - else: - monkeypatch.delitem(sys.modules, "vllm_flash_attn", raising=False) - - # Unsupported head size - backend = get_attn_backend(17, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - def test_invalid_backend(): """Test that invalid attention backend names raise ValueError.""" with ( diff --git a/tests/kernels/attention/test_rocm_attention_selector.py b/tests/kernels/attention/test_rocm_attention_selector.py deleted file mode 100644 index 2a684ed70d..0000000000 --- a/tests/kernels/attention/test_rocm_attention_selector.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest -import torch - -from vllm.config import AttentionConfig, VllmConfig, set_current_vllm_config -from vllm.platforms.rocm import RocmPlatform -from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.attention.selector import _cached_get_attn_backend, get_attn_backend - - -@pytest.fixture(autouse=True) -def clear_cache(): - """Clear lru cache to ensure each test case runs without caching.""" - _cached_get_attn_backend.cache_clear() - - -@pytest.mark.skip(reason="Skipped for now. Should be revisited.") -def test_selector(monkeypatch: pytest.MonkeyPatch): - # Set the current platform to ROCm using monkeypatch - monkeypatch.setattr("vllm.v1.attention.selector.current_platform", RocmPlatform()) - - # Test standard ROCm attention - attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_ATTN) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(16, torch.float16, torch.float16, 16, False) - assert backend.get_name() == "ROCM_FLASH" or backend.get_name() == "TRITON_ATTN" - - # MLA test for deepseek related - # Change the attention backend to triton MLA - attention_config = AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True) - assert backend.get_name() == "TRITON_MLA" - - # If attention backend is None - # If use_mla is true - # The selected backend is triton MLA - attention_config = AttentionConfig(backend=None) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True) - assert backend.get_name() == "TRITON_MLA" - - # Change the attention backend to AITER MLA - attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_AITER_MLA) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 1, False, use_mla=True) - assert backend.get_name() == "ROCM_AITER_MLA" - - # If attention backend is None - # If use_mla is true - # If VLLM_ROCM_USE_AITER is enabled - # The selected backend is ROCM_AITER_MLA - with monkeypatch.context() as m: - m.setenv("VLLM_ROCM_USE_AITER", "1") - - attention_config = AttentionConfig(backend=None) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend( - 576, torch.bfloat16, "auto", 1, False, use_mla=True - ) - assert backend.get_name() == "ROCM_AITER_MLA" diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index a39e1992f3..ef5f29602f 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -1653,19 +1653,6 @@ def test_wna16_linear_gptq_unsupported_config_raises() -> None: INCWNA16LinearScheme(make_layer_config(sym=False)) -def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None: - monkeypatch.setattr(current_platform, "is_xpu", lambda: True) - monkeypatch.setattr(current_platform, "is_cpu", lambda: False) - - with pytest.raises(NotImplementedError, match="unsupported config"): - INCWna16Scheme().get_linear_method( - make_config(weight_bits=2, sym=False), - object(), - "layer", - make_layer_config(bits=2, sym=False), - ) - - def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None: config = make_config(extra_config={"layer": {"bits": 16}}) layer = object.__new__(LinearBase) diff --git a/tests/quantization/test_gptq_v2.py b/tests/quantization/test_gptq_v2.py deleted file mode 100644 index a10088f403..0000000000 --- a/tests/quantization/test_gptq_v2.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests whether vllm correctly load and run gptq_v2 format checkpoints. - -Run `pytest tests/quantization/test_gptq_v2.py --forked`. - -Note: 2/3-bit GPTQ models are no longer supported after the consolidation -to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported. -""" - -import pytest -import torch -from transformers import AutoTokenizer - -from vllm import SamplingParams -from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod - -# A dummy small model quantized by GPTQModel, stored in GPTQ v2 format -# Note: This is a 2-bit model which is no longer supported with Marlin kernels -MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"] - -# Generate multiple sequences for testing, because an 1.7B 2-bit model -# cannot always generate normal texts. -N_SEQ = 5 - - -@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation") -@pytest.mark.parametrize("model_id", MODELS) -def test_model_load(vllm_runner, model_id, monkeypatch): - # `LLM.apply_model` requires pickling a function. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm: - - def check_model(model_id): - for name, submodule in model_id.named_modules(): - # Could check more modules if necessary - if name == "model_id.layers.0.self_attn.qkv_proj": - assert isinstance(submodule.quant_method, AutoGPTQLinearMethod) - # Just break since currently we only check 1 module - break - - # Check if gptq_v2 format is correctly loaded - llm.apply_model(check_model) - - -@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation") -@pytest.mark.parametrize("model_id", MODELS) -def test_model_inference(vllm_runner, model_id): - # Prepare prompt to test the model's generation result. - prompt = "What is the meaning of life?" - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": prompt}, - ] - tokenizer = AutoTokenizer.from_pretrained(model_id) - text = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - enable_thinking=False, # If thinking model, set it to false - ) - sampling_params = SamplingParams( - n=N_SEQ, - max_tokens=128, - temperature=0.7, - top_p=0.8, - top_k=20, - min_p=0, - presence_penalty=2, - ) - - with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm: - # Generate a response to verify inference correctness - output = llm.generate(text, sampling_params) - - # Make sure the output exists - assert output - assert output[0][1] - assert len(output[0][1]) == N_SEQ - - def has_normal_char_distribution(texts, min_len): - for text in texts: - # Response too short - if len(text) < min_len: - return False - - # Basic ratio checks - letters = sum(c.isalpha() for c in text) - spaces = sum(c.isspace() for c in text) - total = len(text) - - letter_ratio = letters / total - space_ratio = spaces / total - - # At least 1 normal text should exist within output sequences - # Normal text should be mostly letters with reasonable spacing - # Some magic numbers, could be adjusted - if 0.5 <= letter_ratio <= 0.9 and 0.01 <= space_ratio <= 0.3: - return True - # No sequence contains normal text, output might be broken - return False - - # Apply some simple checks for giberish output - # Print the output sequences if failed - assert has_normal_char_distribution(output[0][1], 5), output[0][1] diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index 168a0f8978..52167a5073 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -171,42 +171,6 @@ def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch): assert observed_placeholders == [0, 1] -def test_preempt(): - scheduler = create_scheduler(async_scheduling=True) - requests = create_requests(num_requests=10, max_tokens=20) - - for req in requests: - scheduler.add_request(req) - - sched_outputs: deque[SchedulerOutput] = deque() - sched_outputs.append(scheduler.schedule()) - sched_outputs.append(scheduler.schedule()) - - abort_order = [0, 8, 3, 1, 6, 4, 2, 5, 7, 9] - abort_order_copy = abort_order.copy() - - def abort_request(): - if not abort_order: - return - req = requests[abort_order.pop(0)] - scheduler.finish_requests(req.request_id, RequestStatus.FINISHED_ABORTED) - - while sched_outputs: - # Abort a scheduled request. - abort_request() - sched_output = sched_outputs.popleft() - model_runner_output = _make_model_runner_output(sched_output) - scheduler.update_from_output(sched_output, model_runner_output) - - sched_output = scheduler.schedule() - if sched_output.num_scheduled_tokens: - sched_outputs.append(sched_output) - - for i, req in enumerate(requests): - assert req.status == RequestStatus.FINISHED_ABORTED - assert req.num_output_tokens == abort_order_copy.index(i) - - def test_prefix_caching_for_prefill_dedup(): CHUNK_SIZE = 1000 BLOCK_SIZE = 16 diff --git a/tests/v1/core/test_kv_sharing.py b/tests/v1/core/test_kv_sharing.py index ca8787e8d8..4a030c674b 100644 --- a/tests/v1/core/test_kv_sharing.py +++ b/tests/v1/core/test_kv_sharing.py @@ -16,38 +16,6 @@ def new_kv_cache_spec(): ) -def test_initialize_kv_cache_for_kv_sharing_different_attn_groups(): - """Test initializing KV cache sharing with different attention groups. - Layers in the same KV cache group might be placed in different attn groups - if they have different attention backends. - """ - shared_kv_cache_layers = { - "model.layers.2": "model.layers.0", - "model.layers.3": "model.layers.1", - } - - # Layers 0 and 1 both belong in KV cache group 0 - # However, if they have different attention backends, they will be - # placed in different attention groups for KV cache group 0 - kv_cache_groups = [ - KVCacheGroupSpec(["model.layers.0", "model.layers.1"], new_kv_cache_spec()), - ] - - add_kv_sharing_layers_to_kv_cache_groups( - shared_kv_cache_layers=shared_kv_cache_layers, - kv_cache_groups=kv_cache_groups, - ) - - # Check that the layers were added to the correct KV cache group - assert len(kv_cache_groups) == 1 - assert kv_cache_groups[0].layer_names == [ - "model.layers.0", - "model.layers.1", - "model.layers.2", - "model.layers.3", - ] - - def test_initialize_kv_cache_for_kv_sharing_same_attn_groups(): """Test case assuming that all layers in the same KV cache group have the same attention backends. This is true for most models. diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index a691295616..5b3bf1c8be 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -3440,59 +3440,6 @@ class TestEagle: # whole request. assert sched._lookup(req_status) == 4 - def test_eagle_verified_survives_eagle_tighten(self, request_runner): - """Eagle group tightening does NOT clear eagle_verified. - - Groups: 0=non-eagle full-attn, 1=eagle full-attn. - Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2 - (max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared. - Result: 8 tokens (eagle only pops once). - """ - block_size = 4 - groups = [ - KVCacheGroupSpec( - ["layer0"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - ), - is_eagle_group=False, - ), - KVCacheGroupSpec( - ["layer1"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=2, - head_size=1, - dtype=torch.float32, - ), - is_eagle_group=True, - ), - ] - runner = request_runner( - block_size=block_size, - num_gpu_blocks=100, - async_scheduling=False, - kv_cache_groups=groups, - ) - runner.manager.lookup.side_effect = lambda key, req_context: ( - LookupResult.HIT - if int(get_offload_block_hash(key).decode()) in {1, 2, 3} - else LookupResult.MISS - ) - sched = runner.connector_scheduler - req_status = self._make_req_status( - sched, - num_tokens=12, - offload_keys_per_group=[[1, 2, 3], [1, 2, 3]], - ) - # Group 0: 3 hits → max_hit=12, num_hit=12 - # Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8 - # Tightened but IS eagle → no clear. No re-iteration. - assert sched._lookup(req_status) == 8 - # ------------------------------------------------------------------- # Integration tests: store and load via request_runner # ------------------------------------------------------------------- diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index e40d3048c1..c0b9792456 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -167,79 +167,6 @@ def test_topk_impl_equivalence(): assert torch.allclose(result1, result2) -@pytest.mark.skip( - reason="FlashInfer top-k/top-p renorm comparison fails; " - "needs investigation of tolerance threshold or " - "interface differences between Python and FlashInfer implementations" -) -def test_flashinfer_sampler(): - """This test verifies that the FlashInfer top-k and top-p sampling - implementation produces the same results as the Python implementation. - - NOTE: FlashInfer did not directly expose an interface for fused top-k and - top-p prob renorm (it did provide fused sampling but we cannot compare - sampling results due to randomness), so we will compare the probability - renormed consequently by top-k and then top-p of FlashInfer implementation. - """ - try: - from flashinfer.sampling import top_k_renorm_probs, top_p_renorm_probs - - is_flashinfer_available = True - except ImportError: - is_flashinfer_available = False - - FLASHINFER_ENABLED = current_platform.is_cuda() and is_flashinfer_available - - if not FLASHINFER_ENABLED: - pytest.skip("FlashInfer not installed or not available on this platform.") - - torch.set_default_device(DEVICE_TYPE) - generator = Generator(device=DEVICE_TYPE).manual_seed(42) - - # Generate random logits - logits = torch.rand((BATCH_SIZE, VOCAB_SIZE), generator=generator) - - # Generate various top-k and top-p values - k_values = torch.randint(1, 1000, (BATCH_SIZE,), generator=generator) - p_values = ( - torch.rand((BATCH_SIZE,), generator=generator) * 0.5 + 0.5 - ) # range in [0.5, 1.0] - - # Sometimes disable top-k (k=vocab_size) - k_values.masked_fill_( - torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool), - VOCAB_SIZE, - ) - - # Sometimes disable top-p (p=1.0) - p_values.masked_fill_( - torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool), 1.0 - ) - - python_logits = apply_top_k_top_p_pytorch( - logits=logits.clone(), - k=k_values, - p=p_values, - ) - python_probs = torch.softmax(python_logits, dim=-1) - - # FlashInfer only exposed renorm interfaces for probs so convert first - flashinfer_probs = torch.softmax(logits.clone(), dim=-1) - flashinfer_probs = top_k_renorm_probs( - probs=flashinfer_probs, - top_k=k_values, - ) - flashinfer_probs = top_p_renorm_probs( - probs=flashinfer_probs, - top_p=p_values, - ) - - # Compare the results - assert torch.allclose(python_probs, flashinfer_probs, atol=2e-2), ( - "FlashInfer and Python sampling implementations do not match!" - ) - - @pytest.mark.skipif( not FLASHINFER_TOPK_TOPP_SUPPORTED, reason="Requires CUDA and flashinfer", diff --git a/tests/v1/spec_decode/test_backup_token_async_spec.py b/tests/v1/spec_decode/test_backup_token_async_spec.py deleted file mode 100644 index 9340503ea4..0000000000 --- a/tests/v1/spec_decode/test_backup_token_async_spec.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression tests for the backup token fix in prepare_next_token_ids_padded. - -Fixes #38098: with async scheduling, seq_lens_cpu is inflated by unaccepted -draft token placeholders, causing get_token_id() to return -1. -""" - -from __future__ import annotations - -import numpy as np -import pytest -import torch - - -class _FakeRequest: - def __init__(self, prompt_tokens: list[int], output_tokens: list[int]): - self.num_prompt_tokens = len(prompt_tokens) - self._prompt = prompt_tokens - self._output = output_tokens - - @property - def num_tokens(self) -> int: - return self.num_prompt_tokens + len(self._output) - - def get_token_id(self, idx: int) -> int: - if idx < self.num_prompt_tokens: - return self._prompt[idx] - out_idx = idx - self.num_prompt_tokens - if out_idx < len(self._output): - return self._output[out_idx] - return -1 # out of range - - -class _FakeInputBatch: - def __init__( - self, - req_ids: list[str], - num_tokens_no_spec: list[int], - vocab_size: int = 32000, - ): - self.req_ids = req_ids - self.num_reqs = len(req_ids) - self.vocab_size = vocab_size - self.num_tokens_no_spec = np.array(num_tokens_no_spec, dtype=np.int64) - - -def _make_requests( - req_ids: list[str], - prompt_lens: list[int], - output_lens: list[int], -) -> dict[str, _FakeRequest]: - requests = {} - for rid, plen, olen in zip(req_ids, prompt_lens, output_lens): - requests[rid] = _FakeRequest(list(range(plen)), list(range(1000, 1000 + olen))) - return requests - - -def _backup_buggy( - seq_lens_cpu: torch.Tensor, - requests: dict[str, _FakeRequest], - batch: _FakeInputBatch, -) -> list[int]: - """Old logic: uses seq_lens_cpu directly (may be inflated).""" - n = batch.num_reqs - return [ - requests[batch.req_ids[i]].get_token_id(int(seq_lens_cpu[i])) for i in range(n) - ] - - -def _backup_fixed( - requests: dict[str, _FakeRequest], - batch: _FakeInputBatch, -) -> list[int]: - """New logic: uses num_tokens_no_spec - 1 (last committed token).""" - n = batch.num_reqs - idx = (batch.num_tokens_no_spec[:n] - 1).tolist() - return [requests[batch.req_ids[i]].get_token_id(int(idx[i])) for i in range(n)] - - -class TestBackupTokenAsyncSpec: - def test_no_inflation_fixed_returns_last_token(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - # idx = 5-1 = 4 → output[1] = 1001 - assert _backup_fixed(requests, batch) == [1001, 1001] - - def test_inflation_buggy_returns_placeholder(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - # inflated by 3 spec tokens → idx 8 is out of range - seq_lens = torch.tensor([8, 8], dtype=torch.int64) - assert _backup_buggy(seq_lens, requests, batch) == [-1, -1] - - def test_inflation_fixed_returns_correct_token(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - assert _backup_fixed(requests, batch) == [1001, 1001] - - def test_mixed_inflation_per_request(self): - req_ids = ["r0", "r1", "r2"] - requests = { - "r0": _FakeRequest([0, 1], [1000, 1001, 1002]), - "r1": _FakeRequest([0, 1, 2, 3], [2000]), - "r2": _FakeRequest([0], [3000, 3001, 3002, 3003]), - } - batch = _FakeInputBatch(req_ids, [5, 5, 5]) - seq_lens = torch.tensor([7, 9, 5], dtype=torch.int64) - - assert _backup_buggy(seq_lens, requests, batch) == [-1, -1, -1] - assert _backup_fixed(requests, batch) == [1002, 2000, 3003] - - def test_prefill_only_request(self): - """No output tokens yet — backup should be the last prompt token.""" - req_ids = ["r0"] - requests = {"r0": _FakeRequest([10, 20, 30], [])} - batch = _FakeInputBatch(req_ids, [3]) - # idx = 3-1 = 2 → prompt[2] = 30 - assert _backup_fixed(requests, batch) == [30] - - @pytest.mark.parametrize("num_spec_tokens", [1, 2, 3, 4, 5]) - def test_various_spec_token_counts(self, num_spec_tokens: int): - req_ids = ["r0"] - requests = {"r0": _FakeRequest([0, 1, 2], list(range(1000, 1005)))} - batch = _FakeInputBatch(req_ids, [8]) - # idx = 8-1 = 7 → output[4] = 1004 - assert _backup_fixed(requests, batch) == [1004] - - def test_buggy_code_was_always_off_by_one(self): - """The original code used seq_len as index, which is always one past - the end of output_token_ids even without async inflation.""" - req_ids = ["r0"] - requests = {"r0": _FakeRequest([0, 1, 2], [1000, 1001])} - batch = _FakeInputBatch(req_ids, [5]) - - # no inflation: seq_len == num_tokens == 5 → idx 5 is out of range - seq_lens = torch.tensor([5], dtype=torch.int64) - assert _backup_buggy(seq_lens, requests, batch) == [-1] - assert _backup_fixed(requests, batch) == [1001] - - # with inflation: still -1, fixed still correct - seq_lens_inf = torch.tensor([8], dtype=torch.int64) - assert _backup_buggy(seq_lens_inf, requests, batch) == [-1] - assert _backup_fixed(requests, batch) == [1001]