[Refactor] Remove dead or duplicate tests (#58446)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -126,35 +126,6 @@ async def test_score_api_queries_str_n_documents_str_n(
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
|
||||
@@ -143,35 +143,6 @@ async def test_score_api_queries_str_n_documents_str_n(
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
|
||||
@@ -341,61 +341,6 @@ def test_fp32_fallback(device: str):
|
||||
get_attn_backend(16, torch.float32, None)
|
||||
|
||||
|
||||
def test_flash_attn(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test FlashAttn validation."""
|
||||
pytest.skip(
|
||||
"Skipping as current backend selector does not "
|
||||
"handle fallbacks when a backend is explicitly set."
|
||||
)
|
||||
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN)
|
||||
cache_config = CacheConfig(block_size=16)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
# Unsupported CUDA arch
|
||||
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _=None: (7, 5))
|
||||
backend = get_attn_backend(16, torch.float16, None)
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
# Reset the monkeypatch for subsequent tests
|
||||
monkeypatch.undo()
|
||||
|
||||
# Unsupported data type
|
||||
backend = get_attn_backend(16, torch.float8_e4m3fn, None)
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
# Unsupported kv cache data type
|
||||
backend = get_attn_backend(16, torch.float16, "fp8")
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
# Unsupported block size
|
||||
vllm_config.cache_config.block_size = 8
|
||||
backend = get_attn_backend(16, torch.float16, None)
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
# flash-attn is not installed
|
||||
import sys
|
||||
|
||||
vllm_config.cache_config.block_size = 16
|
||||
original_module = sys.modules.get("vllm_flash_attn")
|
||||
monkeypatch.setitem(sys.modules, "vllm_flash_attn", None)
|
||||
backend = get_attn_backend(16, torch.float16, None)
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
# Restore the original module if it existed
|
||||
if original_module is not None:
|
||||
monkeypatch.setitem(sys.modules, "vllm_flash_attn", original_module)
|
||||
else:
|
||||
monkeypatch.delitem(sys.modules, "vllm_flash_attn", raising=False)
|
||||
|
||||
# Unsupported head size
|
||||
backend = get_attn_backend(17, torch.float16, None)
|
||||
assert backend.get_name() != "FLASH_ATTN"
|
||||
|
||||
|
||||
def test_invalid_backend():
|
||||
"""Test that invalid attention backend names raise ValueError."""
|
||||
with (
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import AttentionConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import _cached_get_attn_backend, get_attn_backend
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_cache():
|
||||
"""Clear lru cache to ensure each test case runs without caching."""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Skipped for now. Should be revisited.")
|
||||
def test_selector(monkeypatch: pytest.MonkeyPatch):
|
||||
# Set the current platform to ROCm using monkeypatch
|
||||
monkeypatch.setattr("vllm.v1.attention.selector.current_platform", RocmPlatform())
|
||||
|
||||
# Test standard ROCm attention
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_ATTN)
|
||||
vllm_config = VllmConfig(attention_config=attention_config)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend = get_attn_backend(16, torch.float16, torch.float16, 16, False)
|
||||
assert backend.get_name() == "ROCM_FLASH" or backend.get_name() == "TRITON_ATTN"
|
||||
|
||||
# MLA test for deepseek related
|
||||
# Change the attention backend to triton MLA
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA)
|
||||
vllm_config = VllmConfig(attention_config=attention_config)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True)
|
||||
assert backend.get_name() == "TRITON_MLA"
|
||||
|
||||
# If attention backend is None
|
||||
# If use_mla is true
|
||||
# The selected backend is triton MLA
|
||||
attention_config = AttentionConfig(backend=None)
|
||||
vllm_config = VllmConfig(attention_config=attention_config)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True)
|
||||
assert backend.get_name() == "TRITON_MLA"
|
||||
|
||||
# Change the attention backend to AITER MLA
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_AITER_MLA)
|
||||
vllm_config = VllmConfig(attention_config=attention_config)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend = get_attn_backend(576, torch.bfloat16, "auto", 1, False, use_mla=True)
|
||||
assert backend.get_name() == "ROCM_AITER_MLA"
|
||||
|
||||
# If attention backend is None
|
||||
# If use_mla is true
|
||||
# If VLLM_ROCM_USE_AITER is enabled
|
||||
# The selected backend is ROCM_AITER_MLA
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
attention_config = AttentionConfig(backend=None)
|
||||
vllm_config = VllmConfig(attention_config=attention_config)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend = get_attn_backend(
|
||||
576, torch.bfloat16, "auto", 1, False, use_mla=True
|
||||
)
|
||||
assert backend.get_name() == "ROCM_AITER_MLA"
|
||||
@@ -1653,19 +1653,6 @@ def test_wna16_linear_gptq_unsupported_config_raises() -> None:
|
||||
INCWNA16LinearScheme(make_layer_config(sym=False))
|
||||
|
||||
|
||||
def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None:
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="unsupported config"):
|
||||
INCWna16Scheme().get_linear_method(
|
||||
make_config(weight_bits=2, sym=False),
|
||||
object(),
|
||||
"layer",
|
||||
make_layer_config(bits=2, sym=False),
|
||||
)
|
||||
|
||||
|
||||
def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None:
|
||||
config = make_config(extra_config={"layer": {"bits": 16}})
|
||||
layer = object.__new__(LinearBase)
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests whether vllm correctly load and run gptq_v2 format checkpoints.
|
||||
|
||||
Run `pytest tests/quantization/test_gptq_v2.py --forked`.
|
||||
|
||||
Note: 2/3-bit GPTQ models are no longer supported after the consolidation
|
||||
to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
|
||||
|
||||
# A dummy small model quantized by GPTQModel, stored in GPTQ v2 format
|
||||
# Note: This is a 2-bit model which is no longer supported with Marlin kernels
|
||||
MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
|
||||
|
||||
# Generate multiple sequences for testing, because an 1.7B 2-bit model
|
||||
# cannot always generate normal texts.
|
||||
N_SEQ = 5
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_model_load(vllm_runner, model_id, monkeypatch):
|
||||
# `LLM.apply_model` requires pickling a function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
|
||||
|
||||
def check_model(model_id):
|
||||
for name, submodule in model_id.named_modules():
|
||||
# Could check more modules if necessary
|
||||
if name == "model_id.layers.0.self_attn.qkv_proj":
|
||||
assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
|
||||
# Just break since currently we only check 1 module
|
||||
break
|
||||
|
||||
# Check if gptq_v2 format is correctly loaded
|
||||
llm.apply_model(check_model)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_model_inference(vllm_runner, model_id):
|
||||
# Prepare prompt to test the model's generation result.
|
||||
prompt = "What is the meaning of life?"
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # If thinking model, set it to false
|
||||
)
|
||||
sampling_params = SamplingParams(
|
||||
n=N_SEQ,
|
||||
max_tokens=128,
|
||||
temperature=0.7,
|
||||
top_p=0.8,
|
||||
top_k=20,
|
||||
min_p=0,
|
||||
presence_penalty=2,
|
||||
)
|
||||
|
||||
with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
|
||||
# Generate a response to verify inference correctness
|
||||
output = llm.generate(text, sampling_params)
|
||||
|
||||
# Make sure the output exists
|
||||
assert output
|
||||
assert output[0][1]
|
||||
assert len(output[0][1]) == N_SEQ
|
||||
|
||||
def has_normal_char_distribution(texts, min_len):
|
||||
for text in texts:
|
||||
# Response too short
|
||||
if len(text) < min_len:
|
||||
return False
|
||||
|
||||
# Basic ratio checks
|
||||
letters = sum(c.isalpha() for c in text)
|
||||
spaces = sum(c.isspace() for c in text)
|
||||
total = len(text)
|
||||
|
||||
letter_ratio = letters / total
|
||||
space_ratio = spaces / total
|
||||
|
||||
# At least 1 normal text should exist within output sequences
|
||||
# Normal text should be mostly letters with reasonable spacing
|
||||
# Some magic numbers, could be adjusted
|
||||
if 0.5 <= letter_ratio <= 0.9 and 0.01 <= space_ratio <= 0.3:
|
||||
return True
|
||||
# No sequence contains normal text, output might be broken
|
||||
return False
|
||||
|
||||
# Apply some simple checks for giberish output
|
||||
# Print the output sequences if failed
|
||||
assert has_normal_char_distribution(output[0][1], 5), output[0][1]
|
||||
@@ -171,42 +171,6 @@ def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch):
|
||||
assert observed_placeholders == [0, 1]
|
||||
|
||||
|
||||
def test_preempt():
|
||||
scheduler = create_scheduler(async_scheduling=True)
|
||||
requests = create_requests(num_requests=10, max_tokens=20)
|
||||
|
||||
for req in requests:
|
||||
scheduler.add_request(req)
|
||||
|
||||
sched_outputs: deque[SchedulerOutput] = deque()
|
||||
sched_outputs.append(scheduler.schedule())
|
||||
sched_outputs.append(scheduler.schedule())
|
||||
|
||||
abort_order = [0, 8, 3, 1, 6, 4, 2, 5, 7, 9]
|
||||
abort_order_copy = abort_order.copy()
|
||||
|
||||
def abort_request():
|
||||
if not abort_order:
|
||||
return
|
||||
req = requests[abort_order.pop(0)]
|
||||
scheduler.finish_requests(req.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
|
||||
while sched_outputs:
|
||||
# Abort a scheduled request.
|
||||
abort_request()
|
||||
sched_output = sched_outputs.popleft()
|
||||
model_runner_output = _make_model_runner_output(sched_output)
|
||||
scheduler.update_from_output(sched_output, model_runner_output)
|
||||
|
||||
sched_output = scheduler.schedule()
|
||||
if sched_output.num_scheduled_tokens:
|
||||
sched_outputs.append(sched_output)
|
||||
|
||||
for i, req in enumerate(requests):
|
||||
assert req.status == RequestStatus.FINISHED_ABORTED
|
||||
assert req.num_output_tokens == abort_order_copy.index(i)
|
||||
|
||||
|
||||
def test_prefix_caching_for_prefill_dedup():
|
||||
CHUNK_SIZE = 1000
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
@@ -16,38 +16,6 @@ def new_kv_cache_spec():
|
||||
)
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
|
||||
"""Test initializing KV cache sharing with different attention groups.
|
||||
Layers in the same KV cache group might be placed in different attn groups
|
||||
if they have different attention backends.
|
||||
"""
|
||||
shared_kv_cache_layers = {
|
||||
"model.layers.2": "model.layers.0",
|
||||
"model.layers.3": "model.layers.1",
|
||||
}
|
||||
|
||||
# Layers 0 and 1 both belong in KV cache group 0
|
||||
# However, if they have different attention backends, they will be
|
||||
# placed in different attention groups for KV cache group 0
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(["model.layers.0", "model.layers.1"], new_kv_cache_spec()),
|
||||
]
|
||||
|
||||
add_kv_sharing_layers_to_kv_cache_groups(
|
||||
shared_kv_cache_layers=shared_kv_cache_layers,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
# Check that the layers were added to the correct KV cache group
|
||||
assert len(kv_cache_groups) == 1
|
||||
assert kv_cache_groups[0].layer_names == [
|
||||
"model.layers.0",
|
||||
"model.layers.1",
|
||||
"model.layers.2",
|
||||
"model.layers.3",
|
||||
]
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
|
||||
"""Test case assuming that all layers in the same KV cache group have the same
|
||||
attention backends. This is true for most models.
|
||||
|
||||
@@ -3440,59 +3440,6 @@ class TestEagle:
|
||||
# whole request.
|
||||
assert sched._lookup(req_status) == 4
|
||||
|
||||
def test_eagle_verified_survives_eagle_tighten(self, request_runner):
|
||||
"""Eagle group tightening does NOT clear eagle_verified.
|
||||
|
||||
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
|
||||
Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2
|
||||
(max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared.
|
||||
Result: 8 tokens (eagle only pops once).
|
||||
"""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=False,
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
LookupResult.HIT
|
||||
if int(get_offload_block_hash(key).decode()) in {1, 2, 3}
|
||||
else LookupResult.MISS
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched,
|
||||
num_tokens=12,
|
||||
offload_keys_per_group=[[1, 2, 3], [1, 2, 3]],
|
||||
)
|
||||
# Group 0: 3 hits → max_hit=12, num_hit=12
|
||||
# Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8
|
||||
# Tightened but IS eagle → no clear. No re-iteration.
|
||||
assert sched._lookup(req_status) == 8
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Integration tests: store and load via request_runner
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
@@ -167,79 +167,6 @@ def test_topk_impl_equivalence():
|
||||
assert torch.allclose(result1, result2)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="FlashInfer top-k/top-p renorm comparison fails; "
|
||||
"needs investigation of tolerance threshold or "
|
||||
"interface differences between Python and FlashInfer implementations"
|
||||
)
|
||||
def test_flashinfer_sampler():
|
||||
"""This test verifies that the FlashInfer top-k and top-p sampling
|
||||
implementation produces the same results as the Python implementation.
|
||||
|
||||
NOTE: FlashInfer did not directly expose an interface for fused top-k and
|
||||
top-p prob renorm (it did provide fused sampling but we cannot compare
|
||||
sampling results due to randomness), so we will compare the probability
|
||||
renormed consequently by top-k and then top-p of FlashInfer implementation.
|
||||
"""
|
||||
try:
|
||||
from flashinfer.sampling import top_k_renorm_probs, top_p_renorm_probs
|
||||
|
||||
is_flashinfer_available = True
|
||||
except ImportError:
|
||||
is_flashinfer_available = False
|
||||
|
||||
FLASHINFER_ENABLED = current_platform.is_cuda() and is_flashinfer_available
|
||||
|
||||
if not FLASHINFER_ENABLED:
|
||||
pytest.skip("FlashInfer not installed or not available on this platform.")
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
generator = Generator(device=DEVICE_TYPE).manual_seed(42)
|
||||
|
||||
# Generate random logits
|
||||
logits = torch.rand((BATCH_SIZE, VOCAB_SIZE), generator=generator)
|
||||
|
||||
# Generate various top-k and top-p values
|
||||
k_values = torch.randint(1, 1000, (BATCH_SIZE,), generator=generator)
|
||||
p_values = (
|
||||
torch.rand((BATCH_SIZE,), generator=generator) * 0.5 + 0.5
|
||||
) # range in [0.5, 1.0]
|
||||
|
||||
# Sometimes disable top-k (k=vocab_size)
|
||||
k_values.masked_fill_(
|
||||
torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool),
|
||||
VOCAB_SIZE,
|
||||
)
|
||||
|
||||
# Sometimes disable top-p (p=1.0)
|
||||
p_values.masked_fill_(
|
||||
torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool), 1.0
|
||||
)
|
||||
|
||||
python_logits = apply_top_k_top_p_pytorch(
|
||||
logits=logits.clone(),
|
||||
k=k_values,
|
||||
p=p_values,
|
||||
)
|
||||
python_probs = torch.softmax(python_logits, dim=-1)
|
||||
|
||||
# FlashInfer only exposed renorm interfaces for probs so convert first
|
||||
flashinfer_probs = torch.softmax(logits.clone(), dim=-1)
|
||||
flashinfer_probs = top_k_renorm_probs(
|
||||
probs=flashinfer_probs,
|
||||
top_k=k_values,
|
||||
)
|
||||
flashinfer_probs = top_p_renorm_probs(
|
||||
probs=flashinfer_probs,
|
||||
top_p=p_values,
|
||||
)
|
||||
|
||||
# Compare the results
|
||||
assert torch.allclose(python_probs, flashinfer_probs, atol=2e-2), (
|
||||
"FlashInfer and Python sampling implementations do not match!"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FLASHINFER_TOPK_TOPP_SUPPORTED,
|
||||
reason="Requires CUDA and flashinfer",
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for the backup token fix in prepare_next_token_ids_padded.
|
||||
|
||||
Fixes #38098: with async scheduling, seq_lens_cpu is inflated by unaccepted
|
||||
draft token placeholders, causing get_token_id() to return -1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
class _FakeRequest:
|
||||
def __init__(self, prompt_tokens: list[int], output_tokens: list[int]):
|
||||
self.num_prompt_tokens = len(prompt_tokens)
|
||||
self._prompt = prompt_tokens
|
||||
self._output = output_tokens
|
||||
|
||||
@property
|
||||
def num_tokens(self) -> int:
|
||||
return self.num_prompt_tokens + len(self._output)
|
||||
|
||||
def get_token_id(self, idx: int) -> int:
|
||||
if idx < self.num_prompt_tokens:
|
||||
return self._prompt[idx]
|
||||
out_idx = idx - self.num_prompt_tokens
|
||||
if out_idx < len(self._output):
|
||||
return self._output[out_idx]
|
||||
return -1 # out of range
|
||||
|
||||
|
||||
class _FakeInputBatch:
|
||||
def __init__(
|
||||
self,
|
||||
req_ids: list[str],
|
||||
num_tokens_no_spec: list[int],
|
||||
vocab_size: int = 32000,
|
||||
):
|
||||
self.req_ids = req_ids
|
||||
self.num_reqs = len(req_ids)
|
||||
self.vocab_size = vocab_size
|
||||
self.num_tokens_no_spec = np.array(num_tokens_no_spec, dtype=np.int64)
|
||||
|
||||
|
||||
def _make_requests(
|
||||
req_ids: list[str],
|
||||
prompt_lens: list[int],
|
||||
output_lens: list[int],
|
||||
) -> dict[str, _FakeRequest]:
|
||||
requests = {}
|
||||
for rid, plen, olen in zip(req_ids, prompt_lens, output_lens):
|
||||
requests[rid] = _FakeRequest(list(range(plen)), list(range(1000, 1000 + olen)))
|
||||
return requests
|
||||
|
||||
|
||||
def _backup_buggy(
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
requests: dict[str, _FakeRequest],
|
||||
batch: _FakeInputBatch,
|
||||
) -> list[int]:
|
||||
"""Old logic: uses seq_lens_cpu directly (may be inflated)."""
|
||||
n = batch.num_reqs
|
||||
return [
|
||||
requests[batch.req_ids[i]].get_token_id(int(seq_lens_cpu[i])) for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def _backup_fixed(
|
||||
requests: dict[str, _FakeRequest],
|
||||
batch: _FakeInputBatch,
|
||||
) -> list[int]:
|
||||
"""New logic: uses num_tokens_no_spec - 1 (last committed token)."""
|
||||
n = batch.num_reqs
|
||||
idx = (batch.num_tokens_no_spec[:n] - 1).tolist()
|
||||
return [requests[batch.req_ids[i]].get_token_id(int(idx[i])) for i in range(n)]
|
||||
|
||||
|
||||
class TestBackupTokenAsyncSpec:
|
||||
def test_no_inflation_fixed_returns_last_token(self):
|
||||
req_ids = ["r0", "r1"]
|
||||
requests = _make_requests(req_ids, [3, 3], [2, 2])
|
||||
batch = _FakeInputBatch(req_ids, [5, 5])
|
||||
# idx = 5-1 = 4 → output[1] = 1001
|
||||
assert _backup_fixed(requests, batch) == [1001, 1001]
|
||||
|
||||
def test_inflation_buggy_returns_placeholder(self):
|
||||
req_ids = ["r0", "r1"]
|
||||
requests = _make_requests(req_ids, [3, 3], [2, 2])
|
||||
batch = _FakeInputBatch(req_ids, [5, 5])
|
||||
# inflated by 3 spec tokens → idx 8 is out of range
|
||||
seq_lens = torch.tensor([8, 8], dtype=torch.int64)
|
||||
assert _backup_buggy(seq_lens, requests, batch) == [-1, -1]
|
||||
|
||||
def test_inflation_fixed_returns_correct_token(self):
|
||||
req_ids = ["r0", "r1"]
|
||||
requests = _make_requests(req_ids, [3, 3], [2, 2])
|
||||
batch = _FakeInputBatch(req_ids, [5, 5])
|
||||
assert _backup_fixed(requests, batch) == [1001, 1001]
|
||||
|
||||
def test_mixed_inflation_per_request(self):
|
||||
req_ids = ["r0", "r1", "r2"]
|
||||
requests = {
|
||||
"r0": _FakeRequest([0, 1], [1000, 1001, 1002]),
|
||||
"r1": _FakeRequest([0, 1, 2, 3], [2000]),
|
||||
"r2": _FakeRequest([0], [3000, 3001, 3002, 3003]),
|
||||
}
|
||||
batch = _FakeInputBatch(req_ids, [5, 5, 5])
|
||||
seq_lens = torch.tensor([7, 9, 5], dtype=torch.int64)
|
||||
|
||||
assert _backup_buggy(seq_lens, requests, batch) == [-1, -1, -1]
|
||||
assert _backup_fixed(requests, batch) == [1002, 2000, 3003]
|
||||
|
||||
def test_prefill_only_request(self):
|
||||
"""No output tokens yet — backup should be the last prompt token."""
|
||||
req_ids = ["r0"]
|
||||
requests = {"r0": _FakeRequest([10, 20, 30], [])}
|
||||
batch = _FakeInputBatch(req_ids, [3])
|
||||
# idx = 3-1 = 2 → prompt[2] = 30
|
||||
assert _backup_fixed(requests, batch) == [30]
|
||||
|
||||
@pytest.mark.parametrize("num_spec_tokens", [1, 2, 3, 4, 5])
|
||||
def test_various_spec_token_counts(self, num_spec_tokens: int):
|
||||
req_ids = ["r0"]
|
||||
requests = {"r0": _FakeRequest([0, 1, 2], list(range(1000, 1005)))}
|
||||
batch = _FakeInputBatch(req_ids, [8])
|
||||
# idx = 8-1 = 7 → output[4] = 1004
|
||||
assert _backup_fixed(requests, batch) == [1004]
|
||||
|
||||
def test_buggy_code_was_always_off_by_one(self):
|
||||
"""The original code used seq_len as index, which is always one past
|
||||
the end of output_token_ids even without async inflation."""
|
||||
req_ids = ["r0"]
|
||||
requests = {"r0": _FakeRequest([0, 1, 2], [1000, 1001])}
|
||||
batch = _FakeInputBatch(req_ids, [5])
|
||||
|
||||
# no inflation: seq_len == num_tokens == 5 → idx 5 is out of range
|
||||
seq_lens = torch.tensor([5], dtype=torch.int64)
|
||||
assert _backup_buggy(seq_lens, requests, batch) == [-1]
|
||||
assert _backup_fixed(requests, batch) == [1001]
|
||||
|
||||
# with inflation: still -1, fixed still correct
|
||||
seq_lens_inf = torch.tensor([8], dtype=torch.int64)
|
||||
assert _backup_buggy(seq_lens_inf, requests, batch) == [-1]
|
||||
assert _backup_fixed(requests, batch) == [1001]
|
||||
Reference in New Issue
Block a user