[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests

- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm
  target path; drop the rocm requirement files
- Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules
  and retarget the linear kernel registries away from them
- Reduce platforms/rocm.py to the GCN architecture predicates that live
  code still imports, and delete the RocmPlatform class and plugin
- Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc,
  server_info and sleep routes plus the log_error_stack flag
- Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins;
  the live server process maps none of them
- Delete tests that only build architectures this fork no longer
  registers, and test directories left without any test file
- Restore the tests/v1/kv_connector mock helpers that scheduler tests
  import at module level

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 19:31:07 +07:00
co-authored by Hinata
parent 2066d3143f
commit 5f3c7bd8ab
162 changed files with 599 additions and 27490 deletions
View File
-144
View File
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Callable, Iterable
from typing import Any, NamedTuple
import pytest
import regex as re
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
class Matches(NamedTuple):
# simple pointwise
aiter_rms_quant_fusion: int = 0
rms_quant_fusion: int = 0
act_quant_fusion: int = 0
norm_rope_fusion: int = 0
attn_quant_fusion: int = 0
# distributed
ar_rms_fusion: int = 0
aiter_ar_rms_fusion: int = 0
class ModelFusionInfo(NamedTuple):
model_name: str
matches: Callable[[int], Matches]
"""Given number of hidden layers, produces the matches object"""
model_kwargs: dict[str, Any] = {}
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
class AttentionBackendCase(NamedTuple):
backend: AttentionBackendEnum
model_kwargs: dict[str, Any] = {}
"""Additional args required for attn+quant fusion"""
is_blackwell = lambda: current_platform.is_device_capability_family(100)
"""Are we running on Blackwell, a lot of tests depend on it"""
def nvfp4_kernel_exposes_input_quant_key() -> bool:
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
output format and the cutedsl backend's expected input format.
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
manual fusion.
"""
if not current_platform.is_cuda():
return False
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCuteDslNvFp4LinearKernel,
)
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
if is_supported:
return False
except ImportError:
pass
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
)
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
if is_supported:
return True
except ImportError:
pass
# Fallback: assume no manual fusion support
return False
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
"""Generate all combinations of custom ops for parametrization."""
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
for op_list in itertools.product(*custom_ops_lists):
yield ",".join(op_list)
# Quick inline validation
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
"-quant_fp8,-rms_norm",
"-quant_fp8,+rms_norm",
"+quant_fp8,-rms_norm",
"+quant_fp8,+rms_norm",
]
def has_cuda_graph_wrapper_metadata() -> bool:
from importlib import import_module
try:
module = import_module("torch._inductor.utils")
module.CUDAGraphWrapperMetadata # noqa B018
except AttributeError:
return False
return True
INDUCTOR_GRAPH_PARTITION = [
pytest.param(
True,
marks=pytest.mark.skipif(
not has_cuda_graph_wrapper_metadata(),
reason="torch version does not support Inductor partition",
),
id="inductor_partition",
),
pytest.param(False, id="dynamo_partition"),
]
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
"aiter_rms_quant_fusion": re.compile(
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
),
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
# The combined pass can consume these sites before the standalone pass.
"norm_rope_fusion": re.compile(
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
r"qk_norm_rope_kvcache_fusion.py:\d+] "
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
),
"attn_quant_fusion": re.compile(
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
),
"ar_rms_fusion": re.compile(
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
),
"aiter_ar_rms_fusion": re.compile(
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
),
}
-252
View File
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from collections import defaultdict
import pytest
import regex as re
from vllm import SamplingParams
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
from .common import FUSION_LOG_PATTERNS, AttentionBackendCase, Matches
def run_model(
vllm_runner,
compile_config: CompilationMode | CompilationConfig,
model: str,
**model_kwargs,
):
"""Run a model with the given compilation config for E2E fusion tests."""
compilation_config = (
compile_config
if isinstance(compile_config, CompilationConfig)
else CompilationConfig(mode=compile_config)
)
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
# Allow override from model_kwargs
model_kwargs = {"tensor_parallel_size": 1, **model_kwargs}
model_kwargs = {"disable_custom_all_reduce": True, **model_kwargs}
# No cudagraphs by default
if compilation_config.cudagraph_mode is None:
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
# trust_remote_code/enable_chunked_prefill/block_size/kernel_config are
# pinned to LLM()'s own defaults since VllmRunner overrides them for
# other test needs (e.g. it forces block_size=16, which some attention
# backends like FLASHINFER_MLA reject, and disables JIT warmup).
model_kwargs.setdefault("kernel_config", {"enable_jit_warmup": True})
with vllm_runner(
model,
compilation_config=compilation_config,
trust_remote_code=False,
enable_chunked_prefill=None,
block_size=None,
**model_kwargs,
) as vllm_model:
llm = vllm_model.llm
outputs = llm.generate(prompts, sampling_params)
# Print the outputs.
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
# Get the compile ranges endpoints after vllm config post init
# in order to compute compile ranges correctly
compilation_config.compile_ranges_endpoints = (
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
)
# Fetch match table from each worker via RPC and sum across workers.
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
"get_compilation_match_table"
)
combined: defaultdict[str, int] = defaultdict(int)
for table in worker_tables:
for k, v in table.items():
combined[k] += v
return dict(combined)
@pytest.fixture
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
def run(
model_name: str,
matches: Matches,
model_kwargs: dict,
attn_backend: AttentionBackendCase,
compilation_config: dict,
matches_check: list[str],
use_deepgemm: bool = False,
use_aiter: bool = False,
tp_size: int = 1,
):
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1" if use_aiter else "0")
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
# Filter here to reduce code duplication
backend_name = attn_backend.backend.name.lower()
requires_mla = "deepseek" in model_name.lower()
is_mla = "mla" in backend_name
if requires_mla != is_mla:
pytest.skip(
f"Incompatible model '{model_name}' and "
f"attention backend '{attn_backend.backend.name}'"
)
if backend_name == "rocm_attn" and model_name == "openai/gpt-oss-20b":
pytest.skip(
"ROCM_ATTN does not support attention sinks (required by gpt-oss-20b)"
)
if attn_backend.backend.name == "FLASHINFER":
from vllm.utils.flashinfer import supports_trtllm_attention
if not supports_trtllm_attention():
matches = matches._replace(attn_quant_fusion=0)
# Disable, compile cache to make sure custom passes run.
# Otherwise, we can't verify fusion happened through the logs.
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
# To capture subprocess logs, we need to know whether spawn or fork is used.
# Force spawn as it is more general.
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
model_kwargs = {**attn_backend.model_kwargs, **model_kwargs}
model_kwargs["attention_config"] = {"backend": attn_backend.backend.name}
model_kwargs["tensor_parallel_size"] = tp_size
# Cap warmup memory: tests use small max_model_len (1024) but the
# engine default max_num_batched_tokens is 16384. Warming up large
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
model_kwargs.setdefault("max_num_batched_tokens", 8192)
# Always compile the full graph instead of piecewise
if not compilation_config["use_inductor_graph_partition"]:
compilation_config["splitting_ops"] = []
full_compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.NONE,
mode=CompilationMode.VLLM_COMPILE,
inductor_compile_config={"force_disable_caches": True},
**compilation_config,
)
with caplog_mp_spawn(logging.DEBUG) as log_holder:
match_table = run_model(
vllm_runner, full_compilation_config, model_name, **model_kwargs
)
num_compile_ranges = len(full_compilation_config.get_compile_ranges())
assert num_compile_ranges in [1, 2, 3]
print(f"Compile ranges: {full_compilation_config.get_compile_ranges()}")
print("Fusion results:")
# Iterate through all so printing happens before asserting
log_matches_dict = {}
for match_name, pattern in FUSION_LOG_PATTERNS.items():
log_matches_dict[match_name] = list(pattern.findall(log_holder.text))
print(f"- {match_name}={','.join(log_matches_dict[match_name])}")
# Now check the matches
for match_name in matches_check:
log_matches = list(int(ms) for ms in log_matches_dict[match_name])
# AR+RMS skips the largest range.
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
num_ranges_activated = num_compile_ranges - 1
else:
num_ranges_activated = num_compile_ranges
# TODO: Remove log counting in unit tests
# once all matchers implement VllmFusionPatternMatcherPass
n_expected = tp_size * num_ranges_activated
if match_name not in (
"attn_quant_fusion",
"act_quant_fusion",
"norm_rope_fusion",
):
assert len(log_matches) == n_expected, (
f"Could not find {n_expected} {match_name} "
f"(found {len(log_matches)}) in:\n {log_holder.text}"
)
expected_matches = getattr(matches, match_name)
if match_name == "norm_rope_fusion":
# Opaque LayerName lets the combined pass consume short-range
# sites before the standalone pass; count either owner.
assert len(log_matches) >= n_expected
assert all(m in (0, expected_matches) for m in log_matches)
assert sum(log_matches) == expected_matches * n_expected, (
f"{match_name} expected {expected_matches * n_expected} "
f"sites across both passes, found: {log_matches}"
)
elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
# AR+rms+quant takes precedence over rms+quant if activated.
# That means we get full matching where ar+rms+quant was not
# activated, and less where it was (only the smallest range).
assert sum(m == expected_matches for m in log_matches) == tp_size * (
num_ranges_activated - 1
), "Expecting full rms+quant fusion where ar+rms+quant not activated"
assert all(
expected_matches - matches.ar_rms_fusion <= m <= expected_matches
for m in log_matches
), (
f"Expecting at least {expected_matches - matches.ar_rms_fusion} "
f"where ar+rms+quant was activated"
)
elif match_name == "act_quant_fusion":
actual_match = match_table.get("activation_quant_fusion_pass", 0)
assert actual_match == expected_matches * n_expected, (
f"Could not find {expected_matches * n_expected} "
f"{match_name} (found {actual_match})."
)
elif match_name == "attn_quant_fusion":
actual_match = match_table.get(
"attn_quant_fusion", 0
) + match_table.get("mla_attn_quant_fusion", 0)
assert actual_match == expected_matches * n_expected, (
f"Could not find {expected_matches * n_expected} "
f"{match_name} (found {actual_match})."
)
else:
expected_matches_list = [expected_matches] * n_expected
assert sorted(log_matches) == expected_matches_list, (
f"{match_name} expected: {expected_matches_list}, "
f"found: {sorted(log_matches)}"
)
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
log_matches = re.findall(
r"pass_manager.py:\d+] Skipping "
r".*AllReduceFusionPass.* with compile range",
log_holder.text,
)
n_expected = tp_size * (num_compile_ranges - num_ranges_activated)
assert len(log_matches) == n_expected, (
f'Could not find {n_expected} "Skipping AllReduceFusionPass" '
f"(found {len(log_matches)}) in:\n {log_holder.text}"
)
return run
-51
View File
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from unittest.mock import patch
from vllm.config import VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.v1.engine.async_llm import AsyncLLM
def test_mp_reducer():
"""Test that _reduce_config reducer is registered when AsyncLLM is instantiated
without transformers_modules. This is a regression test for
https://github.com/vllm-project/vllm/pull/18640.
"""
# Ensure transformers_modules is not in sys.modules
if "transformers_modules" in sys.modules:
del sys.modules["transformers_modules"]
with patch("multiprocessing.reducer.register") as mock_register:
engine_args = AsyncEngineArgs(
model="facebook/opt-125m",
max_model_len=32,
gpu_memory_utilization=0.1,
disable_log_stats=True,
)
async_llm = AsyncLLM.from_engine_args(
engine_args,
start_engine_loop=False,
)
assert mock_register.called, (
"multiprocessing.reducer.register should have been called"
)
vllm_config_registered = False
for call_args in mock_register.call_args_list:
# Verify that a reducer for VllmConfig was registered
if len(call_args[0]) >= 2 and call_args[0][0] == VllmConfig:
vllm_config_registered = True
reducer_func = call_args[0][1]
assert callable(reducer_func), "Reducer function should be callable"
break
assert vllm_config_registered, (
"VllmConfig should have been registered to multiprocessing.reducer"
)
async_llm.shutdown()
@@ -1,31 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.entrypoints.llm import LLM
from vllm.sampling_params import SamplingParams
@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
def test_computed_prefix_blocks(model: str):
# This test checks if the engine generates completions both with and
# without optional detokenization, that detokenization includes text
# and no-detokenization doesn't, and that both completions have the same
# token_ids.
prompt = (
"You are a helpful assistant. How do I build a car from cardboard and "
"paper clips? Is there an easy to follow video tutorial available "
"online for free?"
)
llm = LLM(model=model)
sampling_params = SamplingParams(max_tokens=10, temperature=0.0, detokenize=False)
outputs_no_detokenization = llm.generate(prompt, sampling_params)[0].outputs[0]
sampling_params.detokenize = True
outputs_with_detokenization = llm.generate(prompt, sampling_params)[0].outputs[0]
assert outputs_no_detokenization.text == ""
assert outputs_with_detokenization.text != ""
assert outputs_no_detokenization.token_ids == outputs_with_detokenization.token_ids
@@ -1,205 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Orchestration-level integration tests for RayExecutorV2."""
import gc
import os
import pathlib
import pytest
import ray
pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
MODEL = "facebook/opt-125m"
def _get_env_var(worker, name):
return os.environ.get(name)
def _ray_init():
"""Start Ray with the project root on workers' PYTHONPATH.
Without this, workers cannot unpickle actor classes defined in the
``tests`` package, causing FunctionActorManager to fall back to
TemporaryActor which drops async method signatures."""
project_root = str(pathlib.Path(__file__).resolve().parents[2])
ray.init(
ignore_reinit_error=True,
runtime_env={"env_vars": {"PYTHONPATH": project_root}},
)
@pytest.fixture
def ray_init():
_ray_init()
class _AsyncLLMActor:
def start(self, pg, bundle_indices=None, ray_runtime_env=None):
# Needed so collective_rpc can pickle _get_env_var over the
# AsyncLLM -> EngineCore ZMQ boundary.
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
if bundle_indices is not None:
os.environ["VLLM_RAY_BUNDLE_INDICES"] = bundle_indices
else:
os.environ.pop("VLLM_RAY_BUNDLE_INDICES", None)
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.executor.abstract import Executor
engine_args = AsyncEngineArgs(
model=MODEL,
tensor_parallel_size=2,
distributed_executor_backend="ray",
enforce_eager=True,
max_model_len=256,
gpu_memory_utilization=0.8,
)
vllm_config = engine_args.create_engine_config()
vllm_config.parallel_config.placement_group = pg
if ray_runtime_env is not None:
vllm_config.parallel_config.ray_runtime_env = ray_runtime_env
executor_class = Executor.get_class(vllm_config)
self.engine = AsyncLLM(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
log_requests=False,
)
async def generate(self, prompt):
from vllm.sampling_params import SamplingParams
params = SamplingParams(max_tokens=16)
result = None
async for output in self.engine.generate(
prompt, params, request_id="test_request_id"
):
result = output
assert result is not None
return result.outputs[0].text
async def generate_and_get_worker_envs(self, prompt, env_names):
from vllm.sampling_params import SamplingParams
params = SamplingParams(max_tokens=16)
result = None
async for output in self.engine.generate(
prompt, params, request_id="test_request_id"
):
result = output
assert result is not None
text = result.outputs[0].text
env_results = {}
for name in env_names:
vals = await self.engine.collective_rpc(
_get_env_var, timeout=10, args=(name,)
)
env_results[name] = vals
return text, env_results
def shutdown(self):
if engine := getattr(self, "engine", None):
engine.shutdown()
del self.engine
gc.collect()
AsyncLLMActor = ray.remote(num_cpus=0, max_concurrency=1)(_AsyncLLMActor)
def test_multi_replicas(ray_init):
pg1 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
pg2 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
ray.get([pg1.ready(), pg2.ready()])
actor1 = AsyncLLMActor.remote()
actor2 = AsyncLLMActor.remote()
ray.get(actor1.start.remote(pg1))
ray.get(actor2.start.remote(pg2))
out1, out2 = ray.get(
[
actor1.generate.remote("Hello world"),
actor2.generate.remote("Hello world"),
]
)
assert len(out1) > 0
assert len(out2) > 0
def test_multi_replicas_with_bundle_indices(ray_init):
pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 4, strategy="PACK")
ray.get(pg.ready())
actor1 = AsyncLLMActor.remote()
actor2 = AsyncLLMActor.remote()
ray.get(actor1.start.remote(pg, bundle_indices="2,1"))
ray.get(actor2.start.remote(pg, bundle_indices="0,3"))
out1, out2 = ray.get(
[
actor1.generate.remote("Hello world"),
actor2.generate.remote("Hello world"),
]
)
assert len(out1) > 0
assert len(out2) > 0
def test_env_var_and_runtime_env_propagation():
"""Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
propagate to RayWorkerProc actors.
"""
sentinel_vars = {
"NCCL_DEBUG": "INFO",
"HF_TOKEN": "test_sentinel_token",
}
for k, v in sentinel_vars.items():
os.environ[k] = v
try:
# Called directly (not via the ray_init fixture) because sentinel
# env vars must be in os.environ before ray.init() so that Ray
# worker processes inherit them.
_ray_init()
pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
ray.get(pg.ready())
# Include the project root so that RayWorkerProc actors can
# unpickle _get_env_var.
project_root = str(pathlib.Path(__file__).resolve().parents[2])
ray_runtime_env = {
"env_vars": {
"RAY_RUNTIME_ENV_TEST": "ray_runtime_env",
"PYTHONPATH": project_root,
},
}
actor = AsyncLLMActor.remote()
ray.get(actor.start.remote(pg, ray_runtime_env=ray_runtime_env))
all_env_names = list(sentinel_vars) + ["RAY_RUNTIME_ENV_TEST"]
text, env_results = ray.get(
actor.generate_and_get_worker_envs.remote("Hello world", all_env_names)
)
assert len(text) > 0
for name, expected in sentinel_vars.items():
for val in env_results[name]:
assert val == expected
for val in env_results["RAY_RUNTIME_ENV_TEST"]:
assert val == "ray_runtime_env"
finally:
for k in sentinel_vars:
os.environ.pop(k, None)
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import os
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
from tests.v1.utils import check_request_balancing
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
DP_SIZE = os.getenv("DP_SIZE", "1")
@pytest.fixture(scope="module")
def default_server_args():
return [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
"--api-server-count",
"4",
"--data_parallel_size",
DP_SIZE,
]
@pytest.fixture(scope="module")
def server(default_server_args):
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_single_completion(
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
) -> None:
async def make_request():
completion = await client.completions.create(
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
)
assert completion.id is not None
assert completion.choices is not None and len(completion.choices) == 1
choice = completion.choices[0]
# The exact number of tokens can vary slightly with temperature=1.0,
# so we check for a reasonable minimum length.
assert len(choice.text) >= 1
# Finish reason might not always be 'length' if the model finishes early
# or due to other reasons, especially with high temperature.
# So, we'll accept 'length' or 'stop'.
assert choice.finish_reason in ("length", "stop")
# Token counts can also vary, so we check they are positive.
assert completion.usage.completion_tokens > 0
assert completion.usage.prompt_tokens > 0
assert completion.usage.total_tokens > 0
return completion
# Test single request
result = await make_request()
assert result is not None
await asyncio.sleep(0.5)
# Send two bursts of requests
num_requests = 100
tasks = [make_request() for _ in range(num_requests)]
results = await asyncio.gather(*tasks)
assert len(results) == num_requests
assert all(completion is not None for completion in results)
await asyncio.sleep(0.5)
tasks = [make_request() for _ in range(num_requests)]
results = await asyncio.gather(*tasks)
assert len(results) == num_requests
assert all(completion is not None for completion in results)
# Check request balancing via Prometheus metrics if DP_SIZE > 1
check_request_balancing(server, int(DP_SIZE))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_completion_streaming(
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
) -> None:
prompt = "What is an LLM?"
async def make_streaming_request():
# Perform a non-streaming request to get the expected full output
single_completion = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
)
single_output = single_completion.choices[0].text
# Perform the streaming request
stream = await client.completions.create(
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
)
chunks: list[str] = []
finish_reason_count = 0
last_chunk = None
async for chunk in stream:
chunks.append(chunk.choices[0].text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
last_chunk = chunk # Keep track of the last chunk
# finish reason should only return in the last block for OpenAI API
assert finish_reason_count == 1, "Finish reason should appear exactly once."
assert last_chunk is not None, "Stream should have yielded at least one chunk."
assert last_chunk.choices[0].finish_reason == "length", (
"Finish reason should be 'length'."
)
# Check that the combined text matches the non-streamed version.
assert "".join(chunks) == single_output, (
"Streamed output should match non-streamed output."
)
return True # Indicate success for this request
# Test single request
result = await make_streaming_request()
assert result is not None
await asyncio.sleep(0.5)
# Send two bursts of requests
num_requests = 100
tasks = [make_streaming_request() for _ in range(num_requests)]
results = await asyncio.gather(*tasks)
assert len(results) == num_requests, (
f"Expected {num_requests} results, got {len(results)}"
)
assert all(results), "Not all streaming requests completed successfully."
await asyncio.sleep(0.5)
tasks = [make_streaming_request() for _ in range(num_requests)]
results = await asyncio.gather(*tasks)
assert len(results) == num_requests, (
f"Expected {num_requests} results, got {len(results)}"
)
assert all(results), "Not all streaming requests completed successfully."
# Check request balancing via Prometheus metrics if DP_SIZE > 1
check_request_balancing(server, int(DP_SIZE))
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
@pytest.fixture(scope="function")
def vision_llm(vllm_runner):
with vllm_runner(
"microsoft/Phi-3.5-vision-instruct",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
trust_remote_code=True,
limit_mm_per_prompt={"image": 2},
seed=0,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize(
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
)
def test_chat_multi_image(vision_llm, image_urls: list[str]):
messages = [
{
"role": "user",
"content": [
*(
{"type": "image_url", "image_url": {"url": image_url}}
for image_url in image_urls
),
{"type": "text", "text": "What's in this image?"},
],
}
]
outputs = vision_llm.chat(messages)
assert len(outputs) >= 0
@@ -1,643 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
MODEL_NAME_SHORT = "gpt2"
BASE_MODEL_PATHS = [
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT),
]
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
task = "generate"
runner_type = "generate"
model = MODEL_NAME
tokenizer = MODEL_NAME
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
revision = None
code_revision = None
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
hf_text_config = MockHFConfig()
logits_processors: list[str] | None = None
diff_sampling_param: dict | None = None
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
encoder_config = None
generation_config: str = "auto"
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
renderer_num_workers: int = 1
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
def _build_renderer(model_config: MockModelConfig):
return HfRenderer(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
cached_tokenizer_from_config(model_config),
)
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_chat = OpenAIServingChat(
engine,
models,
response_role="assistant",
online_renderer=online_renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_chat.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_chat
@pytest.mark.asyncio
async def test_chat_error_non_stream():
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=None,
logprobs=None,
finish_reason="error",
)
request_output = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
async def mock_generate(*args, **kwargs):
yield request_output
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=False,
)
with pytest.raises(GenerationError):
await serving_chat.create_chat_completion(request)
@pytest.mark.asyncio
async def test_openai_chat_keeps_mm_cache_for_engine_execution():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_chat.render_chat_request(request)
assert isinstance(result, tuple)
assert (
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is False
)
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_chat_error_stream(include_usage: bool):
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
completion_output_1 = CompletionOutput(
index=0,
text="Hello",
token_ids=[100],
cumulative_logprob=None,
logprobs=None,
finish_reason=None,
)
request_output_1 = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output_1],
finished=False,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
completion_output_2 = CompletionOutput(
index=0,
text="Hello",
token_ids=[100],
cumulative_logprob=None,
logprobs=None,
finish_reason="error",
)
request_output_2 = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output_2],
finished=True,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
async def mock_generate(*args, **kwargs):
yield request_output_1
yield request_output_2
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_chat.create_chat_completion(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert len(chunks) >= 2
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_chat_error_stream_first_chunk(include_usage: bool):
"""An error before the first token must reach the streaming client."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=None,
logprobs=None,
finish_reason="error",
)
request_output = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
async def mock_generate(*args, **kwargs):
yield request_output
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_chat.create_chat_completion(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert len(chunks) >= 2
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.parametrize(
"image_content",
[
[{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}],
[{"image_url": {"url": "https://example.com/image.jpg"}}],
],
)
def test_system_message_warns_on_image(image_content):
"""Test that system messages with image content trigger a warning."""
with patch(
"vllm.entrypoints.openai.chat_completion.protocol.logger"
) as mock_logger:
ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{
"role": "system",
"content": image_content,
}
],
)
mock_logger.warning_once.assert_called()
call_args = str(mock_logger.warning_once.call_args)
assert "System messages should only contain text" in call_args
assert "image_url" in call_args
def test_system_message_accepts_text():
"""Test that system messages can contain text content."""
# Should not raise an exception
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
],
)
assert request.messages[0]["role"] == "system"
def test_system_message_accepts_text_array():
"""Test that system messages can contain an array with text content."""
# Should not raise an exception
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{
"role": "system",
"content": [{"type": "text", "text": "You are a helpful assistant."}],
},
],
)
assert request.messages[0]["role"] == "system"
def test_user_message_accepts_image():
"""Test that user messages can still contain image content."""
# Should not raise an exception
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.jpg"},
},
],
},
],
)
assert request.messages[0]["role"] == "user"
@pytest.mark.parametrize(
"audio_content",
[
[
{
"type": "input_audio",
"input_audio": {"data": "base64data", "format": "wav"},
}
],
[{"input_audio": {"data": "base64data", "format": "wav"}}],
],
)
def test_system_message_warns_on_audio(audio_content):
"""Test that system messages with audio content trigger a warning."""
with patch(
"vllm.entrypoints.openai.chat_completion.protocol.logger"
) as mock_logger:
ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{
"role": "system",
"content": audio_content,
}
],
)
mock_logger.warning_once.assert_called()
call_args = str(mock_logger.warning_once.call_args)
assert "System messages should only contain text" in call_args
assert "input_audio" in call_args
@pytest.mark.parametrize(
"video_content",
[
[{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}],
[{"video_url": {"url": "https://example.com/video.mp4"}}],
],
)
def test_system_message_warns_on_video(video_content):
"""Test that system messages with video content trigger a warning."""
with patch(
"vllm.entrypoints.openai.chat_completion.protocol.logger"
) as mock_logger:
ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{
"role": "system",
"content": video_content,
}
],
)
mock_logger.warning_once.assert_called()
call_args = str(mock_logger.warning_once.call_args)
assert "System messages should only contain text" in call_args
assert "video_url" in call_args
def test_json_schema_response_format_missing_schema():
"""When response_format type is 'json_schema' but the json_schema field
is not provided, request construction should raise a validation error
so the API returns 400 instead of 500."""
with pytest.raises(Exception, match="json_schema.*must be provided"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "json_schema"},
)
@pytest.mark.asyncio
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
mm_processor_kwargs={
"patch_size": 1,
"vision_min_num_patches": 3_000_000_000,
},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
@pytest.mark.asyncio
async def test_online_renderer_rejects_media_io_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
media_io_kwargs={"video": {"num_frames": 1_000_000}},
)
with pytest.raises(
VLLMValidationError,
match="Per-request media_io_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
validate_request_mm_kwargs(
mm_processor_kwargs={"use_audio_in_video": True},
media_io_kwargs={"video": {"num_frames": 4}},
trust_request_mm_kwargs=True,
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
with pytest.raises(
VLLMValidationError,
match="Invalid response_format structural_tag",
):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "structural_tag", "format": format_value},
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_batch_structural_tag_response_format_invalid(format_value):
"""Batch chat should reject malformed structural tags at request parsing."""
with pytest.raises(
VLLMValidationError,
match="Invalid response_format structural_tag",
):
BatchChatCompletionRequest(
model=MODEL_NAME,
messages=[[{"role": "user", "content": "hello"}]],
response_format={"type": "structural_tag", "format": format_value},
)
@pytest.mark.parametrize(
"structural_tag",
[
"not json",
"",
# json.loads raises a plain ValueError, not JSONDecodeError
'{"type": "structural_tag", "x": ' + "1" * 5000 + "}",
],
)
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
VLLMValidationError,
match="Invalid structured_outputs structural_tag",
):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
structured_outputs={"structural_tag": structural_tag},
)
@pytest.mark.parametrize("field", ["response_format", "structured_outputs"])
def test_deeply_nested_structural_tag_rejected(field):
"""Structural tags are compiled during request parsing, so a deeply nested
one must be rejected there instead of blocking the server while converting,
and reported as too deep rather than malformed."""
schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200
tag = (
'{"type": "structural_tag", "format": {"type": "json_schema", '
f'"json_schema": {schema}}}}}'
)
value = json.loads(tag) if field == "response_format" else {"structural_tag": tag}
with pytest.raises(VLLMValidationError, match="nested too deeply"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
**{field: value},
)
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
**{field_name: "2"},
)
@@ -1,304 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import openai # use the official client for correctness check
import pybase64 as base64
import pytest
import pytest_asyncio
import torch
# downloading lora to test lora requests
from openai import BadRequestError
from transformers import AutoConfig
from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "facebook/opt-125m"
LORA_SERVING_MODEL_NAME = "opt125m-lora"
CONFIG = AutoConfig.from_pretrained(MODEL_NAME)
@pytest.fixture(scope="module", params=["use-lora"])
def default_server_args(
request: pytest.FixtureRequest, opt125_lora_files: str
) -> list[str]:
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
# Prompt Embeds server args
"--enable-prompt-embeds",
]
if request.param == "use-lora":
lora_module_1 = {
"name": LORA_SERVING_MODEL_NAME,
"path": opt125_lora_files,
"base_model_name": MODEL_NAME,
}
args.extend(
[
"--enable-lora",
"--lora-module",
json.dumps(lora_module_1),
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
)
return args
EXAMPLE_PROMPTS = [
"Hello, my name is",
"What is an LLM?",
]
def _encode_embeds(embeds: torch.Tensor):
buffer = io.BytesIO()
torch.save(embeds, buffer)
return base64.b64encode(buffer.getvalue()).decode("utf-8")
@pytest.fixture(scope="module")
def example_prompt_embeds(hf_runner):
"""Create example embeddings and return them as base64 encoded string."""
with hf_runner(MODEL_NAME) as hf_model:
example_embeddings = hf_model.get_prompt_embeddings(EXAMPLE_PROMPTS)
return [_encode_embeds(item) for item in example_embeddings]
@pytest.fixture(scope="module")
def server_with_prompt_embeds(default_server_args):
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client_with_prompt_embeds(server_with_prompt_embeds):
async with server_with_prompt_embeds.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
async def test_completions_with_prompt_embeds(
example_prompt_embeds,
client_with_prompt_embeds: openai.AsyncOpenAI,
model_name: str,
):
encoded_embeds, encoded_embeds2 = example_prompt_embeds
# Test case: Single prompt embeds input
completion = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": encoded_embeds},
)
assert len(completion.choices[0].text) >= 1
assert completion.choices[0].prompt_logprobs is None
# Test case: batch completion with prompt_embeds
completion = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
)
assert len(completion.choices) == 2
assert len(completion.choices[0].text) >= 1
assert len(completion.choices[1].text) >= 1
# Test case: streaming with prompt_embeds
single_completion = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": encoded_embeds},
)
single_output = single_completion.choices[0].text
stream = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
stream=True,
extra_body={"prompt_embeds": encoded_embeds},
)
chunks = []
finish_reason_count = 0
async for chunk in stream:
chunks.append(chunk.choices[0].text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
assert finish_reason_count == 1
assert chunk.choices[0].finish_reason == "length"
assert chunk.choices[0].text
assert "".join(chunks) == single_output
# Test case: batch streaming with prompt_embeds
stream = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
stream=True,
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
)
chunks_stream_embeds: list[list[str]] = [[], []]
finish_reason_count = 0
async for chunk in stream:
chunks_stream_embeds[chunk.choices[0].index].append(chunk.choices[0].text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
assert finish_reason_count == 2
assert chunk.choices[0].finish_reason == "length"
assert chunk.choices[0].text
assert len(chunks_stream_embeds[0]) > 0
assert len(chunks_stream_embeds[1]) > 0
# Test case: mixed text and prompt_embeds
completion_mixed = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt="This is a prompt",
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": encoded_embeds},
)
assert len(completion.choices) == 2
completion_text_only = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt="This is a prompt",
max_tokens=5,
temperature=0.0,
)
completion_embeds_only = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": encoded_embeds},
)
# Embeddings responses should be handled first
assert completion_mixed.choices[0].text == completion_embeds_only.choices[0].text
assert completion_mixed.choices[1].text == completion_text_only.choices[0].text
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
async def test_completions_errors_with_prompt_embeds(
client_with_prompt_embeds: openai.AsyncOpenAI, model_name: str
):
# Test error case: invalid prompt_embeds
with pytest.raises(BadRequestError):
await client_with_prompt_embeds.completions.create(
prompt=None,
model=model_name,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": "invalid_base64"},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("logprobs_arg", [1, 0])
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
async def test_completions_with_logprobs_and_prompt_embeds(
example_prompt_embeds,
client_with_prompt_embeds: openai.AsyncOpenAI,
logprobs_arg: int,
model_name: str,
):
encoded_embeds, encoded_embeds2 = example_prompt_embeds
# Test case: Logprobs using prompt_embeds
completion = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
echo=False,
logprobs=logprobs_arg,
extra_body={"prompt_embeds": encoded_embeds},
)
logprobs = completion.choices[0].logprobs
assert logprobs is not None
assert len(logprobs.text_offset) == 5
assert len(logprobs.token_logprobs) == 5
assert len(logprobs.top_logprobs) == 5
for top_logprobs in logprobs.top_logprobs[1:]:
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
assert len(logprobs.tokens) == 5
# Test case: Log probs with batch completion and prompt_embeds
completion = await client_with_prompt_embeds.completions.create(
model=model_name,
prompt=None,
max_tokens=5,
temperature=0.0,
echo=False,
logprobs=logprobs_arg,
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
)
assert len(completion.choices) == 2
for choice in completion.choices:
logprobs = choice.logprobs
assert logprobs is not None
assert len(logprobs.text_offset) == 5
assert len(logprobs.token_logprobs) == 5
assert len(logprobs.top_logprobs) == 5
for top_logprobs in logprobs.top_logprobs[1:]:
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
assert len(logprobs.tokens) == 5
@pytest.mark.asyncio
async def test_prompt_logprobs_raises_error(
example_prompt_embeds,
client_with_prompt_embeds: openai.AsyncOpenAI,
):
encoded_embeds, _ = example_prompt_embeds
with pytest.raises(BadRequestError, match="not compatible"):
await client_with_prompt_embeds.completions.create(
model=MODEL_NAME,
prompt=None,
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": encoded_embeds, "prompt_logprobs": True},
)
@pytest.mark.asyncio
async def test_empty_prompt_embeds(
client_with_prompt_embeds: openai.AsyncOpenAI,
) -> None:
await client_with_prompt_embeds.completions.create(
model=MODEL_NAME,
prompt="Hello",
max_tokens=5,
temperature=0.0,
extra_body={"prompt_embeds": []},
)
@@ -1,256 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import suppress
from dataclasses import dataclass, field
from http import HTTPStatus
from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
MOCK_RESOLVER_NAME = "mock_test_resolver"
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
"""Minimal mock ModelConfig for testing."""
model: str = MODEL_NAME
runner_type = "generate"
tokenizer: str = MODEL_NAME
trust_remote_code: bool = False
tokenizer_mode: str = "auto"
max_model_len: int = 100
tokenizer_revision: str | None = None
multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
logits_processors: list[str] | None = None
diff_sampling_param: dict | None = None
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
encoder_config = None
generation_config: str = "auto"
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
renderer_num_workers: int = 1
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
class MockLoRAResolver(LoRAResolver):
async def resolve_lora(
self, base_model_name: str, lora_name: str
) -> LoRARequest | None:
if lora_name == "test-lora":
return LoRARequest(
lora_name="test-lora",
lora_int_id=1,
lora_path="/fake/path/test-lora",
)
elif lora_name == "invalid-lora":
return LoRARequest(
lora_name="invalid-lora",
lora_int_id=2,
lora_path="/fake/path/invalid-lora",
)
return None
@pytest.fixture(autouse=True)
def register_mock_resolver():
"""Fixture to register and unregister the mock LoRA resolver."""
resolver = MockLoRAResolver()
LoRAResolverRegistry.register_resolver(MOCK_RESOLVER_NAME, resolver)
yield
# Cleanup: remove the resolver after the test runs
if MOCK_RESOLVER_NAME in LoRAResolverRegistry.resolvers:
del LoRAResolverRegistry.resolvers[MOCK_RESOLVER_NAME]
def _build_renderer(model_config: MockModelConfig):
return HfRenderer(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
cached_tokenizer_from_config(model_config),
)
@pytest.fixture
def mock_serving_setup():
"""Provides a mocked engine and serving completion instance."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
async def mock_add_lora_side_effect(lora_request: LoRARequest):
"""Simulate engine behavior when adding LoRAs."""
if lora_request.lora_name == "test-lora":
# Simulate successful addition
return True
if lora_request.lora_name == "invalid-lora":
# Simulate failure during addition (e.g. invalid format)
raise ValueError(f"Simulated failure adding LoRA: {lora_request.lora_name}")
return True
mock_engine.add_lora = AsyncMock(side_effect=mock_add_lora_side_effect)
async def mock_generate(*args, **kwargs):
for _ in []:
yield _
mock_engine.generate = MagicMock(spec=AsyncLLM.generate, side_effect=mock_generate)
mock_engine.generate.reset_mock()
mock_engine.add_lora.reset_mock()
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
models = OpenAIServingModels(
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_completion = OpenAIServingCompletion(
mock_engine, models, online_renderer=online_renderer, request_logger=None
)
return mock_engine, serving_completion
@pytest.mark.asyncio
async def test_serving_completion_with_lora_resolver(mock_serving_setup, monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
mock_engine, serving_completion = mock_serving_setup
lora_model_name = "test-lora"
req_found = CompletionRequest(
model=lora_model_name,
prompt="Generate with LoRA",
)
# Suppress potential errors during the mocked generate call,
# as we are primarily checking for add_lora and generate calls
with suppress(Exception):
await serving_completion.create_completion(req_found)
mock_engine.add_lora.assert_awaited_once()
called_lora_request = mock_engine.add_lora.call_args[0][0]
assert isinstance(called_lora_request, LoRARequest)
assert called_lora_request.lora_name == lora_model_name
mock_engine.generate.assert_called_once()
called_lora_request = mock_engine.generate.call_args[1]["lora_request"]
assert isinstance(called_lora_request, LoRARequest)
assert called_lora_request.lora_name == lora_model_name
@pytest.mark.asyncio
async def test_serving_completion_resolver_not_found(mock_serving_setup, monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
mock_engine, serving_completion = mock_serving_setup
non_existent_model = "non-existent-lora-adapter"
req = CompletionRequest(
model=non_existent_model,
prompt="what is 1+1?",
)
response = await serving_completion.create_completion(req)
mock_engine.add_lora.assert_not_awaited()
mock_engine.generate.assert_not_called()
assert isinstance(response, ErrorResponse)
assert response.error.code == HTTPStatus.NOT_FOUND.value
assert non_existent_model in response.error.message
@pytest.mark.asyncio
async def test_serving_completion_resolver_add_lora_fails(
mock_serving_setup, monkeypatch
):
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
mock_engine, serving_completion = mock_serving_setup
invalid_model = "invalid-lora"
req = CompletionRequest(
model=invalid_model,
prompt="what is 1+1?",
)
response = await serving_completion.create_completion(req)
# Assert add_lora was called before the failure
mock_engine.add_lora.assert_awaited_once()
called_lora_request = mock_engine.add_lora.call_args[0][0]
assert isinstance(called_lora_request, LoRARequest)
assert called_lora_request.lora_name == invalid_model
# Assert generate was *not* called due to the failure
mock_engine.generate.assert_not_called()
# Assert the correct error response
assert isinstance(response, ErrorResponse)
assert response.error.code == HTTPStatus.BAD_REQUEST.value
assert invalid_model in response.error.message
@pytest.mark.asyncio
async def test_serving_completion_flag_not_set(mock_serving_setup):
mock_engine, serving_completion = mock_serving_setup
lora_model_name = "test-lora"
req_found = CompletionRequest(
model=lora_model_name,
prompt="Generate with LoRA",
)
await serving_completion.create_completion(req_found)
mock_engine.add_lora.assert_not_called()
mock_engine.generate.assert_not_called()
@@ -396,7 +396,6 @@ def server_with_store(default_server_args):
default_server_args,
env_dict={
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
"VLLM_SERVER_DEV_MODE": "1",
},
) as remote_server:
yield remote_server
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
import requests
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
class TestWorkerExtension:
def get_model_name(self) -> str:
"""Test non-pydantic return type."""
return MODEL_NAME
def echo_args_kwargs(self, *args, **kwargs) -> dict[str, Any]:
"""Echo back both args and kwargs."""
return dict(
args=list(args),
kwargs=kwargs,
total_items=len(args) + len(kwargs),
)
def return_none(self, *args, **kwargs) -> None:
"""Test method that does not return anything."""
return
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"8192",
"--max-num-seqs",
"128",
"--worker-extension-cls",
"tests.entrypoints.serve.dev.rpc.test_collective_rpc.TestWorkerExtension",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
) as remote_server:
yield remote_server
def test_get_model_name(server):
"""Test basic response."""
response = requests.post(
server.url_for("collective_rpc"), json={"method": "get_model_name"}
)
assert response.status_code == 200
results = response.json()
assert "results" in results
assert results["results"] == [MODEL_NAME]
def test_return_none(server):
"""Test return none."""
response = requests.post(
server.url_for("collective_rpc"), json={"method": "return_none"}
)
assert response.status_code == 200
results = response.json()
assert results["results"] == [None]
def test_echo_args_kwargs(server):
"""Test args, kwargs, and dict response."""
args = ["arg1", "arg2"]
kwargs = {"key1": "value1", "key2": "value2"}
response = requests.post(
server.url_for("collective_rpc"),
json={"method": "echo_args_kwargs", "args": args, "kwargs": kwargs},
)
assert response.status_code == 200
results = response.json()
result = results["results"][0]
assert result["args"] == args
assert result["kwargs"] == kwargs
assert result["total_items"] == len(args) + len(kwargs)
@@ -70,19 +70,17 @@ class MockModelConfig:
@pytest.fixture(params=get_args(SupportedTask))
def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]:
def task_routes(request) -> tuple[str, list[tuple[str, list[str]]]]:
"""For each supported task, build an app with only that task's routers,
extract all routes, and return the task name and routes."""
task = request.param
# Enable development mode to register all routes (including dev-only routes).
monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1")
app = FastAPI()
args = Namespace()
app.state = Namespace()
app.state.args = args
# Register routers for this specific task (development mode already enabled).
# Register routers for this specific task.
register_api_routers(
args,
app,
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.tokenize.api_router import attach_router
from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeChatRequest,
TokenizeCompletionRequest,
TokenizeResponse,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
BASE_MODEL_PATHS = [
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
]
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
task = "generate"
runner_type = "generate"
model = MODEL_NAME
tokenizer = MODEL_NAME
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
hf_text_config = MockHFConfig()
logits_processors: list[str] | None = None
diff_sampling_param: dict | None = None
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
encoder_config = None
generation_config: str = "auto"
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
renderer_num_workers: int = 1
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
return ServingTokenization(
models,
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
)
@pytest.mark.asyncio
async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign]
return_value=(
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
)
request = TokenizeChatRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
assert (
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@pytest.mark.asyncio
async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign]
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
request = TokenizeCompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
)
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
assert (
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
class TestDetokenizeClientErrorResponses:
"""Client-caused errors from /detokenize are 4xx, not 500.
/detokenize used to wrap ``create_detokenize`` in a blanket
``except Exception`` that reported every failure as 500, while its
sibling /tokenize let exceptions propagate to the global exception
handlers, which map client-caused errors to 4xx and keep genuine
server errors at 500. These tests pin the shared contract for both
endpoints. See #52246 for the same fix in the Anthropic entrypoint.
"""
@staticmethod
def _make_api_app(handler: MagicMock) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(log_error_stack=False)
app.state.serving_tokenization = handler
attach_router(app)
init_exception_handler(app)
return app
def _post(self, handler: MagicMock, path: str = "/detokenize"):
app = self._make_api_app(handler)
body: dict[str, Any] = {"model": "test-model"}
if path == "/detokenize":
body["tokens"] = [1, 2, 3]
else:
body["prompt"] = "Hello"
with TestClient(app, raise_server_exceptions=False) as client:
return client.post(path, json=body)
def test_vllm_validation_error_returns_bad_request(self):
handler = MagicMock(spec=ServingTokenization)
handler.create_detokenize.side_effect = VLLMValidationError(
"invalid token ids", parameter="tokens"
)
response = self._post(handler)
assert response.status_code == HTTPStatus.BAD_REQUEST
error = response.json()["error"]
assert error["type"] == "BadRequestError"
assert error["param"] == "tokens"
def test_vllm_not_found_error_returns_not_found(self):
handler = MagicMock(spec=ServingTokenization)
handler.create_detokenize.side_effect = VLLMNotFoundError(
"LoRA adapter nonexistent-lora not found"
)
response = self._post(handler)
assert response.status_code == HTTPStatus.NOT_FOUND
assert response.json()["error"]["type"] == "NotFoundError"
def test_value_error_returns_bad_request(self):
handler = MagicMock(spec=ServingTokenization)
handler.create_detokenize.side_effect = ValueError("bad input")
response = self._post(handler)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json()["error"]["type"] == "BadRequestError"
def test_overflow_error_returns_bad_request(self):
"""Fast tokenizers raise OverflowError for out-of-range token ids.
This used to be special-cased into a RequestValidationError; the
global OverflowError handler keeps the same 400 status code.
"""
handler = MagicMock(spec=ServingTokenization)
handler.create_detokenize.side_effect = OverflowError(
"out of range integral type conversion attempted"
)
response = self._post(handler)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json()["error"]["type"] == "BadRequestError"
def test_server_error_still_returns_internal_server_error(self):
"""Genuine server bugs keep the existing 500 behaviour."""
handler = MagicMock(spec=ServingTokenization)
handler.create_detokenize.side_effect = RuntimeError("boom")
response = self._post(handler)
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert response.json()["error"]["type"] == "InternalServerError"
def test_tokenize_and_detokenize_agree_on_client_errors(self):
"""The two sibling endpoints report the same client error alike."""
statuses = {}
for path, method in (
("/tokenize", "create_tokenize"),
("/detokenize", "create_detokenize"),
):
handler = MagicMock(spec=ServingTokenization)
getattr(handler, method).side_effect = VLLMValidationError(
"invalid input", parameter="model"
)
statuses[path] = self._post(handler, path=path).status_code
assert statuses["/tokenize"] == HTTPStatus.BAD_REQUEST
assert statuses["/detokenize"] == HTTPStatus.BAD_REQUEST
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
View File
-27
View File
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.utils import jit_monitor
@pytest.fixture(autouse=True)
def _reset_monitor():
"""Reset global monitor state between tests.
``activate()`` installs process-global hooks and flips module globals, so
without this every test would inherit the previous test's monitor state.
"""
def reset():
jit_monitor._active = False
jit_monitor._mode = "warn"
jit_monitor._verbose = False
jit_monitor._cutedsl_hook_installed = False
jit_monitor._tilelang_hook_installed = False
jit_monitor._tilelang_jitimpl_compile_depth = 0
reset()
yield
reset()
-19
View File
@@ -1,19 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.utils.torch_utils import (
create_kv_caches_with_random,
create_kv_caches_with_random_flash,
)
@pytest.fixture()
def kv_cache_factory():
return create_kv_caches_with_random
@pytest.fixture()
def kv_cache_factory_flashinfer():
return create_kv_caches_with_random_flash
View File
View File
@@ -1,158 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from .common import Config
from .mk_objects import (
MK_ALL_PREPARE_FINALIZE_TYPES,
MK_FUSED_EXPERT_TYPES,
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES,
)
def make_config_arg_parser(description: str):
def to_pf_class_type(s: str) -> mk.FusedMoEPrepareAndFinalizeModular:
for pf in MK_ALL_PREPARE_FINALIZE_TYPES:
if pf.__name__ == s:
return pf
raise ValueError(f"Cannot find a PrepareFinalize type that matches {s}")
def to_experts_class_type(s: str) -> mk.FusedMoEExpertsModular:
for fe in MK_FUSED_EXPERT_TYPES:
if fe.__name__ == s:
return fe
raise ValueError(f"Cannot find a FusedExperts type that matches {s}")
def to_quant_torch_dtype(s: str) -> torch.dtype:
if s == "torch.float8_e4m3fn":
return torch.float8_e4m3fn
raise ValueError(f"Unsupported quant type {s}")
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"--world-size",
type=int,
default=2,
help="Number of ranks that participate in all2all",
)
parser.add_argument(
"--pf-type",
type=to_pf_class_type,
required=True,
help=(
"Choose a PrepareFinalize Type : "
f"{[x.__name__ for x in MK_ALL_PREPARE_FINALIZE_TYPES]}"
),
)
parser.add_argument(
"--experts-type",
type=to_experts_class_type,
required=True,
help=(
f"Choose a FusedExpert type : {[x.__name__ for x in MK_FUSED_EXPERT_TYPES]}"
),
)
parser.add_argument(
"-m",
nargs="+",
type=int,
default=[64],
help="num tokens per rank",
)
parser.add_argument(
"-k",
type=int,
default=7168,
help="hidden-size",
)
parser.add_argument(
"-n",
type=int,
default=1024,
help="N dimension of the first fused-moe matmul",
)
parser.add_argument(
"--num-experts", type=int, default=32, help="Global num experts"
)
parser.add_argument("--topk", nargs="+", type=int, default=[4, 1], help="num topk")
# Quant args
parser.add_argument(
"--quant-dtype", type=to_quant_torch_dtype, help="Quant datatype"
)
parser.add_argument(
"--per-token-quantized-activations",
action="store_true",
help=("The input activations must be per-token quantized"),
)
parser.add_argument(
"--per-channel-quantized-weights",
action="store_true",
help="The weights must be per-channel quantized.",
)
parser.add_argument(
"--block-shape", nargs="+", type=int, help="Quantization block shape"
)
# Torch trace profile generation args
parser.add_argument(
"--torch-trace-dir-path",
type=str,
default=None,
help="Get torch trace for single execution",
)
return parser
def _validate_args(args: argparse.Namespace):
if args.quant_dtype is not None:
assert args.quant_dtype == torch.float8_e4m3fn
if args.block_shape is not None:
assert len(args.block_shape) == 2, (
f"block shape must have 2 elements. got {args.block_shape}"
)
if args.experts_type in MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES:
assert args.world_size == 1, "Single GPU objects need world size set to 1"
if args.torch_trace_dir_path is not None:
from pathlib import Path
assert Path(args.torch_trace_dir_path).is_dir(), (
f"Please create {args.torch_trace_dir_path}"
)
def make_config(args: argparse.Namespace) -> Config:
_validate_args(args)
quant_config = None
if args.quant_dtype is not None:
quant_config = FusedMoEQuantConfig.make(
quant_dtype=args.quant_dtype,
per_act_token_quant=args.per_token_quantized_activations,
per_out_ch_quant=args.per_channel_quantized_weights,
block_shape=args.block_shape,
)
return Config(
Ms=args.m,
K=args.k,
N=args.n,
E=args.num_experts,
topks=args.topk,
dtype=torch.bfloat16, # hard-code
quant_config=quant_config,
prepare_finalize_type=args.pf_type,
fused_experts_type=args.experts_type,
world_size=args.world_size,
torch_trace_dir_path=args.torch_trace_dir_path,
)
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from enum import Enum
from itertools import product
import torch
from tqdm import tqdm
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.config import FUSED_MOE_UNQUANTIZED_CONFIG
from vllm.utils.torch_utils import set_random_seed
from .common import (
Config,
RankTensors,
WeightTensors,
reference_moe_impl,
run_modular_kernel,
)
from .mk_objects import (
MK_FUSED_EXPERT_TYPES,
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES,
MK_QUANT_CONFIGS,
)
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
class Result(Enum):
PASS = 1
FAIL = 2
SKIP = 3
def rank_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
config: Config,
weights: WeightTensors,
):
set_random_seed(pgi.rank)
# get weights to this device
weights.to_current_device()
Ms = config.Ms
assert isinstance(Ms, list)
TOPKs = config.topks
assert isinstance(TOPKs, list)
for m, topk in product(Ms, TOPKs):
print(f"Running m={m}, topk={topk} ...")
# override m and topk
cfgx = copy.deepcopy(config)
cfgx.Ms = m
cfgx.topks = topk
# inputs for rank
rank_tensors = RankTensors.make(cfgx, pgi)
# modular kernel out
mk_out = run_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
with set_current_vllm_config(vllm_config):
ref_out = reference_moe_impl(cfgx, weights, rank_tensors)
torch.testing.assert_close(ref_out, mk_out, atol=3e-2, rtol=3e-2)
def make_feature_matrix(csv_file_path: str):
from dataclasses import asdict
import pandas as pd
def add_to_results(
config: Config, success: Result, results_df: pd.DataFrame | None = None
):
config_dict = asdict(config)
config_dict["prepare_finalize_type"] = config_dict[
"prepare_finalize_type"
].__name__
config_dict["fused_experts_type"] = config_dict["fused_experts_type"].__name__
config_dict["per_tensor_act_quant"] = config.is_per_tensor_act_quant
quant_config_dict = config_dict["quant_config"]
del config_dict["quant_config"]
if quant_config_dict is None:
quant_config = FUSED_MOE_UNQUANTIZED_CONFIG
quant_config_dict = asdict(quant_config)
config_dict |= quant_config_dict
result_dict = config_dict | {"success": success.name}
result_df = pd.DataFrame([result_dict])
if results_df is None:
results_df = result_df
else:
results_df = pd.concat([results_df, result_df], ignore_index=True)
return results_df
Ms = [64]
Ks = [7168] # hidden sizes
Ns = [2048]
TOPKs = [[4, 1]]
Es = [32]
DTYPEs = [torch.bfloat16]
PF_TYPES = MK_MULTI_GPU_PREPARE_FINALIZE_TYPES
FE_TYPES = MK_FUSED_EXPERT_TYPES
Q_TYPES = MK_QUANT_CONFIGS
combinations = list(
product(Ms, Ks, Ns, Es, TOPKs, DTYPEs, PF_TYPES, FE_TYPES, Q_TYPES)
)
results_df: pd.DataFrame | None = None
for m, k, n, e, topks, dtype, pf_type, experts_type, quant_config in tqdm(
combinations
):
config = Config(
Ms=[m],
K=k,
N=n,
E=e,
topks=topks,
dtype=dtype,
prepare_finalize_type=pf_type,
fused_experts_type=experts_type,
quant_config=quant_config,
world_size=2,
)
success = None
if config.is_valid()[0]:
print(f"Running config : {config.describe()} ...")
try:
weights: WeightTensors = WeightTensors.make(config)
vllm_config, env_dict = config.make_env_data()
parallel_launch_with_config(
config.world_size,
rank_worker,
vllm_config,
env_dict,
config,
weights,
)
success = Result.PASS
except Exception as _:
success = Result.FAIL
else:
success = Result.SKIP
results_df = add_to_results(config, success, results_df)
if results_df is not None:
results_df.to_csv(f"{csv_file_path}")
if __name__ == "__main__":
import argparse
from pathlib import Path
parser = argparse.ArgumentParser(
description=(
"Make ModularKernel feature matrix \n"
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.make_feature_matrix " # noqa: E501
"-f ./feature_matrices/feature_matrix.csv"
)
)
parser.add_argument(
"-f",
"--feature-matrix-csv-file-path",
type=str,
required=True,
help="File name to Generate a .csv file",
)
args = parser.parse_args()
csv_path = args.feature_matrix_csv_file_path
assert csv_path.endswith("csv"), (
f"Need a file path ending with .csv, got {csv_path}"
)
assert Path(csv_path).parent.is_dir(), (
f"Cannot find parent directory for {Path(csv_path).parent}"
)
make_feature_matrix(args.feature_matrix_csv_file_path)
@@ -1,182 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Callable
from itertools import product
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
from .common import (
Config,
RankTensors,
WeightTensors,
_make_gscale,
make_modular_kernel,
)
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
def do_profile(
fn: Callable,
fn_kwargs: dict[Any, Any],
pgi: ProcessGroupInfo,
config: Config,
num_warmups: int = 5,
):
for _ in range(num_warmups):
fn(**fn_kwargs)
with torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
with_stack=True,
record_shapes=True,
) as tprof:
fn(**fn_kwargs)
device = torch.accelerator.current_device_index()
torch.accelerator.synchronize(device)
# TODO (varun): Add a descriptive trace file name
tprof.export_chrome_trace(
f"{config.torch_trace_dir_path}/m{config.M}_{pgi.rank}_trace.json"
)
def profile_modular_kernel(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
config: Config,
weights: WeightTensors,
rank_tensors: RankTensors,
) -> None:
assert isinstance(config.Ms, int)
assert isinstance(config.topks, int)
# weights for rank
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
if config.quant_dtype == "nvfp4":
gscale = _make_gscale(config.num_local_experts)
else:
gscale = None
quant_config = FusedMoEQuantConfig.make(
config.quant_dtype,
w1_scale=rank_weights.w1_scale,
w2_scale=rank_weights.w2_scale,
a1_scale=rank_tensors.hidden_states_scale,
g1_alphas=(1 / rank_weights.w1_gs) if rank_weights.w1_gs is not None else None,
g2_alphas=(1 / rank_weights.w2_gs) if rank_weights.w2_gs is not None else None,
a1_gscale=gscale,
a2_gscale=gscale,
block_shape=config.quant_block_shape,
per_act_token_quant=config.is_per_act_token_quant,
per_out_ch_quant=config.is_per_out_ch_quant,
)
# make modular kernel
mk = make_modular_kernel(config, vllm_config, quant_config)
topk_ids = rank_tensors.topk_ids.to(
mk.prepare_finalize.topk_indices_dtype() or rank_tensors.topk_ids.dtype
)
# impls might update the tensor in place
hidden_states = rank_tensors.hidden_states.clone()
mk_kwargs = {
"hidden_states": hidden_states,
"w1": rank_weights.w1,
"w2": rank_weights.w2,
"topk_weights": rank_tensors.topk_weights,
"topk_ids": topk_ids,
"activation": MoEActivation.SILU,
"expert_map": rank_tensors.expert_map,
"global_num_experts": config.E,
"apply_router_weight_on_input": config.topk == 1
and config.supports_apply_weight_on_input(),
}
num_tokens = hidden_states.shape[0]
num_tokens_across_dp = torch.tensor(
[num_tokens] * config.world_size, device="cpu", dtype=torch.int
)
with set_forward_context(
None,
vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
):
do_profile(mk.apply, mk_kwargs, pgi, config)
def rank_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
config: Config,
weights: WeightTensors,
):
set_random_seed(pgi.rank)
# workspace manager is normally initialized by GPUModelRunner; we initialize
# it here for the standalone benchmark process.
init_workspace_manager(torch.device(f"cuda:{pgi.local_rank}"))
# get weights to this device
weights.to_current_device()
Ms = config.Ms
assert isinstance(Ms, list)
TOPKs = config.topks
assert isinstance(TOPKs, list)
for m, topk in product(Ms, TOPKs):
print(f"Running m={m}, topk={topk} ...")
# override m and topk
cfgx = copy.deepcopy(config)
cfgx.Ms = m
cfgx.topks = topk
# inputs for rank
rank_tensors = RankTensors.make(cfgx, pgi)
profile_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
def run(config: Config):
weights: WeightTensors = WeightTensors.make(config)
vllm_config, env_dict = config.make_env_data()
parallel_launch_with_config(
config.world_size, rank_worker, vllm_config, env_dict, config, weights
)
if __name__ == "__main__":
from .cli_args import make_config, make_config_arg_parser
parser = make_config_arg_parser(
description=(
"Run single prepare-finalize & fused-experts combination test"
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.profile_modular_kernel " # noqa: E501
"--pf-type DeepEPLLPrepareAndFinalize --experts-type BatchedTritonExperts"
)
)
args = parser.parse_args()
assert args.torch_trace_dir_path is not None, (
"Please pass in a directory to store torch traces"
)
config = make_config(args)
run(config)
@@ -1,259 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Platform-agnostic eligibility tests for GateLinear's ROCm router GEMMs.
These assert the dispatch flags directly, so they run device-free by mocking
the platform predicates. ``allow_cublas_router_gemm`` selects the
bf16xbf16->fp32 ``torch.mm`` epilogue, while ``allow_fp32_router_gemm`` selects
the gfx950 low-M kernel with fp32 weights and output.
The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
term; a biased gate must fall back so the bias is not silently dropped.
"""
import inspect
import pytest
import torch
import vllm.model_executor.layers.fused_moe.router.gate_linear as gate_linear_mod
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
def _make_gate(
monkeypatch,
*,
is_rocm: bool,
is_cuda: bool = False,
bias: bool = False,
params_dtype: torch.dtype = torch.bfloat16,
out_dtype: torch.dtype | None = torch.float32,
input_size: int = 2048,
output_size: int = 64,
on_gfx950: bool = False,
parallel_world_size: int = 1,
) -> GateLinear:
"""Build a GateLinear with platform predicates mocked, no GPU needed."""
for target in (
"vllm.model_executor.layers.linear",
"vllm.model_executor.parameter",
):
monkeypatch.setattr(
f"{target}.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
f"{target}.get_tensor_model_parallel_world_size",
lambda: parallel_world_size,
)
platform = gate_linear_mod.current_platform
monkeypatch.setattr(platform, "is_cuda", lambda: is_cuda)
monkeypatch.setattr(platform, "is_rocm", lambda: is_rocm)
monkeypatch.setattr(platform, "is_device_capability", lambda *a, **k: False)
monkeypatch.setattr(platform, "is_device_capability_family", lambda *a, **k: False)
if is_rocm:
import vllm.platforms.rocm as rocm_platform
monkeypatch.setattr(rocm_platform, "on_gfx950", lambda: on_gfx950)
return GateLinear(
input_size=input_size,
output_size=output_size,
bias=bias,
out_dtype=out_dtype,
params_dtype=params_dtype,
)
def test_rocm_no_bias_bf16_fp32_enables_fused_gemm(monkeypatch):
gate = _make_gate(monkeypatch, is_rocm=True, bias=False)
assert not gate.allow_specialized_router_gemm
assert gate.allow_cublas_router_gemm
def test_rocm_bias_disables_fused_gemm(monkeypatch):
# torch.mm cannot add a bias, so a biased gate must not take the fused path.
gate = _make_gate(monkeypatch, is_rocm=True, bias=True)
assert not gate.allow_cublas_router_gemm
def test_rocm_fp32_weight_disables_fused_gemm(monkeypatch):
gate = _make_gate(monkeypatch, is_rocm=True, params_dtype=torch.float32)
assert not gate.allow_cublas_router_gemm
def test_rocm_non_fp32_out_dtype_disables_fused_gemm(monkeypatch):
gate = _make_gate(monkeypatch, is_rocm=True, out_dtype=torch.bfloat16)
assert not gate.allow_cublas_router_gemm
def test_non_rocm_non_cuda_disables_fused_gemm(monkeypatch):
# Neither the CUDA specialized path nor the ROCm branch applies.
gate = _make_gate(monkeypatch, is_rocm=False, is_cuda=False)
assert not gate.allow_cublas_router_gemm
def test_rocm_set_out_dtype_enables_fused_gemm(monkeypatch):
gate = _make_gate(monkeypatch, is_rocm=True, bias=False, out_dtype=None)
assert not gate.allow_cublas_router_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_cublas_router_gemm
def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
gate = _make_gate(monkeypatch, is_rocm=True, bias=True, out_dtype=None)
gate.set_out_dtype(torch.float32)
assert not gate.allow_cublas_router_gemm
@pytest.mark.parametrize(
("input_size", "output_size"),
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
)
def test_rocm_gfx950_enables_fp32_router_gemm(
monkeypatch, input_size: int, output_size: int
) -> None:
gate = _make_gate(
monkeypatch,
is_rocm=True,
params_dtype=torch.float32,
input_size=input_size,
output_size=output_size,
on_gfx950=True,
)
assert gate.allow_fp32_router_gemm
assert not gate.allow_cublas_router_gemm
@pytest.mark.parametrize("ep_size", [2, 4, 8])
def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
monkeypatch, ep_size: int
) -> None:
gate = _make_gate(
monkeypatch,
is_rocm=True,
params_dtype=torch.float32,
input_size=6144,
output_size=128,
on_gfx950=True,
parallel_world_size=ep_size,
)
assert gate.weight.shape == (128, 6144)
assert gate.allow_fp32_router_gemm
@pytest.mark.parametrize(
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
[
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
],
)
def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
monkeypatch,
input_size: int,
output_size: int,
params_dtype: torch.dtype,
bias: bool,
on_gfx950: bool,
) -> None:
gate = _make_gate(
monkeypatch,
is_rocm=True,
params_dtype=params_dtype,
bias=bias,
input_size=input_size,
output_size=output_size,
on_gfx950=on_gfx950,
)
assert not gate.allow_fp32_router_gemm
# ---------------------------------------------------------------------------
# ROCm BF16x3 tier (fp32 router weights)
# ---------------------------------------------------------------------------
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
"""Build a gate with the ROCm bf16x3 platform check mocked."""
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm as rocm_bf16x3,
)
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
kwargs.setdefault("params_dtype", torch.float32)
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch)
assert gate.allow_rocm_bf16x3_router_gemm
# Built by process_weights_after_loading, not lazily in forward().
# Registered up front but still None, so named_buffers() skips it.
assert gate._bf16x3_weight is None
assert "_bf16x3_weight" in gate._buffers
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
# torch.mm has no bias term.
gate = _make_bf16x3_gate(monkeypatch, bias=True)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
gate = _make_bf16x3_gate(monkeypatch, supported=False)
assert not gate.allow_rocm_bf16x3_router_gemm
assert not gate._rocm_bf16x3_weight_eligible
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
"""An ineligible out_dtype must not allocate a split that is never read;
it is 1.5x the router weight on every layer."""
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
gate.quant_method.process_weights_after_loading(gate)
assert gate._bf16x3_weight is None
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
assert not gate.allow_rocm_bf16x3_router_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_dispatch_is_a_custom_op():
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
assert out.shape == (8192, 128)
assert out.dtype == torch.float32
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
"""The gfx950 low-M fp32 tier claims these shapes too and returns
unconditionally, so bf16x3 is only reachable if checked first."""
gate = _make_bf16x3_gate(
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
)
assert gate.allow_fp32_router_gemm
assert gate.allow_rocm_bf16x3_router_gemm
source = inspect.getsource(type(gate).forward)
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
"allow_fp32_router_gemm"
)
-112
View File
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import time
import pytest
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.launchers.api_server.entry import (
build_async_engine_client_from_engine_args,
)
from vllm.inputs import TextPrompt
from vllm.lora.request import LoRARequest
from vllm.sampling_params import SamplingParams
from vllm.utils.async_utils import merge_async_iterators
MODEL_PATH = "zai-org/chatglm3-6b"
LORA_RANK = 64
DEFAULT_MAX_LORAS = 4 * 3
def get_lora_requests(lora_path) -> list[LoRARequest]:
lora_requests: list[LoRARequest] = [
LoRARequest(lora_name=f"{i}", lora_int_id=i, lora_path=lora_path)
for i in range(1, DEFAULT_MAX_LORAS + 1)
]
return lora_requests
async def requests_processing_time(llm, lora_requests: list[LoRARequest]) -> float:
sampling_params = SamplingParams(
n=1, temperature=0.0, top_p=1.0, ignore_eos=True, max_tokens=1
)
generators = []
start = time.perf_counter()
for lora_request in lora_requests:
lora_int_id = lora_request.lora_int_id
generator = llm.generate(
prompt=TextPrompt(prompt=f"hello {lora_int_id}", multi_modal_data=None), # type: ignore
sampling_params=sampling_params,
lora_request=lora_request,
request_id=f"test{lora_int_id}",
)
generators.append(generator)
all_gens = merge_async_iterators(*generators)
async for i, res in all_gens:
pass
end = time.perf_counter()
return end - start
@pytest.mark.asyncio
async def test_add_lora(chatglm3_lora_files):
"""The add_lora function is used to preload some LoRA adapters into the
engine in anticipation of future requests using these adapters. To test
this functionality, we use the async engine to process some requests - We
do it twice, once with add_lora() preloading and once without.
We measure the request processing time in both cases and expect the time
to be lesser in the case with add_lora() calls.
"""
lora_requests: list[LoRARequest] = get_lora_requests(chatglm3_lora_files)
max_loras = len(set([lr.lora_int_id for lr in lora_requests]))
# Create engine in eager-mode. Due to high max_loras, the CI can
# OOM during cuda-graph capture.
engine_args = AsyncEngineArgs(
model=MODEL_PATH,
enable_lora=True,
max_loras=max_loras,
max_lora_rank=LORA_RANK,
max_model_len=128,
gpu_memory_utilization=0.8, # avoid OOM
trust_remote_code=True,
enforce_eager=True,
)
# split lora_requests into 3 parts
part_size = len(lora_requests) // 3
dummy_run_requests = lora_requests[:part_size]
warmup_run_requests = lora_requests[part_size : part_size * 2]
cold_run_requests = lora_requests[part_size * 2 :]
async with build_async_engine_client_from_engine_args(engine_args) as llm:
# Dummy run - So any 1-time functionality like triton kernel compilation
# is complete here.
await requests_processing_time(llm, dummy_run_requests)
# Run with warmup
add_lora_tasks = [llm.add_lora(lr) for lr in warmup_run_requests]
add_lora_results = await asyncio.gather(*add_lora_tasks)
# Test that all all_lora calls are successful.
assert all(add_lora_results)
time_with_add_lora = await requests_processing_time(llm, warmup_run_requests)
# Run without any warmup
time_cold_start = await requests_processing_time(llm, cold_run_requests)
print(f"time hot-start {time_with_add_lora} vs time cold-start {time_cold_start} ")
assert time_with_add_lora < time_cold_start, (
f"time_with_add_lora={time_with_add_lora}, "
f"time_cold_start={time_cold_start}"
"The engine request processing time with LoRA pre-loading "
"must be less than the version that does on-demand LoRA loading."
)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
from vllm import LLM, EngineArgs
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.model_loader import tensorizer as tensorizer_mod
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from vllm.utils.network_utils import get_file_store_init_method
from vllm.v1.executor import UniProcExecutor
from vllm.v1.worker.worker_base import WorkerWrapperBase
MODEL_REF = "facebook/opt-125m"
@pytest.fixture()
def model_ref():
return MODEL_REF
@pytest.fixture(autouse=True)
def allow_insecure_serialization(monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
@pytest.fixture(autouse=True)
def cleanup():
cleanup_dist_env_and_memory(shutdown_ray=True)
@pytest.fixture()
def just_serialize_model_tensors(model_ref, monkeypatch, tmp_path):
def noop(*args, **kwargs):
return None
args = EngineArgs(model=model_ref)
tc = TensorizerConfig(tensorizer_uri=f"{tmp_path}/model.tensors")
monkeypatch.setattr(tensorizer_mod, "serialize_extra_artifacts", noop)
tensorizer_mod.tensorize_vllm_model(args, tc)
yield tmp_path
@pytest.fixture(autouse=True)
def tensorizer_config():
config = TensorizerConfig(tensorizer_uri="vllm")
return config
@pytest.fixture()
def model_path(model_ref, tmp_path):
yield tmp_path / model_ref / "model.tensors"
def assert_from_collective_rpc(engine: LLM, closure: Callable, closure_kwargs: dict):
res = engine.collective_rpc(method=closure, kwargs=closure_kwargs)
return all(res)
# This is an object pulled from tests/v1/engine/test_engine_core.py
# Modified to strip the `load_model` method from its `_init_executor`
# method. It's purely used as a dummy utility to run methods that test
# Tensorizer functionality
class DummyExecutor(UniProcExecutor):
def _init_executor(self) -> None:
"""Initialize the worker and load the model."""
self.driver_worker = WorkerWrapperBase(rpc_rank=0)
distributed_init_method = get_file_store_init_method()
local_rank = 0
# set local rank as the device index if specified
device_info = self.vllm_config.device_config.device.__str__().split(":")
if len(device_info) > 1:
local_rank = int(device_info[1])
rank = 0
is_driver_worker = True
kwargs = dict(
vllm_config=self.vllm_config,
local_rank=local_rank,
rank=rank,
distributed_init_method=distributed_init_method,
is_driver_worker=is_driver_worker,
)
self.mm_receiver_cache = None
self.collective_rpc("init_worker", args=([kwargs],))
self.collective_rpc("init_device")
def shutdown(self):
if hasattr(self, "thread_pool"):
self.thread_pool.shutdown(wait=False)
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-17
View File
@@ -1,17 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
@pytest.fixture(autouse=True)
def require_rocm_cdna4() -> None:
if not current_platform.is_rocm():
pytest.skip("requires ROCm")
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("requires CDNA4")
-41
View File
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared BF16 baselines for the quantization comparison tests.
Several tests here quantize the same model and compare against an
unquantized run of it. Computing that reference once per session avoids
booting an identical engine for every quantization scheme under test.
Only the outputs are cached; each engine is still torn down as usual, so
tests keep the isolation ``VllmRunner.__exit__`` provides.
"""
import pytest
@pytest.fixture(scope="session")
def quant_baseline_logprobs(vllm_runner):
"""Greedy logprobs for an unquantized model, computed once per config."""
cache: dict = {}
def baseline(
model: str,
prompts: list[str],
*,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
):
key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts))
if key not in cache:
with vllm_runner(
model,
max_model_len=max_model_len,
enforce_eager=True,
) as vllm_model:
cache[key] = vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
return cache[key]
return baseline
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from setuptools import setup
setup(
name="vllm_add_dummy_model",
version="0.1",
packages=["vllm_add_dummy_model"],
entry_points={
"vllm.general_plugins": ["register_dummy_model = vllm_add_dummy_model:register"]
},
)
@@ -1,18 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from setuptools import setup
setup(
name="vllm_add_dummy_platform",
version="0.1",
packages=["vllm_add_dummy_platform"],
entry_points={
"vllm.platform_plugins": [
"dummy_platform_plugin = vllm_add_dummy_platform:dummy_platform_plugin" # noqa
],
"vllm.general_plugins": [
"dummy_custom_ops = vllm_add_dummy_platform:register_ops"
],
},
)
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
def dummy_platform_plugin() -> str | None:
# Real platform plugins often import vLLM modules while registering.
from vllm.utils.torch_utils import direct_register_custom_op # noqa: F401
return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
def register_ops():
import vllm_add_dummy_platform.dummy_custom_ops # noqa
@@ -1,10 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
class DummyAttentionBackend(TritonAttentionBackend):
@staticmethod
def get_name() -> str:
return "Dummy_Backend"
@@ -1,32 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.platforms.interface import Platform, PlatformEnum
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import AttentionSelectorConfig
else:
VllmConfig = None
class DummyPlatform(Platform):
_enum = PlatformEnum.OOT
device_name = "DummyDevice"
device_type: str = "privateuseone"
dispatch_key: str = "PrivateUse1"
@classmethod
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
vllm_config.compilation_config.custom_ops = ["all"]
@classmethod
def get_attn_backend_cls(
cls,
selected_backend: "AttentionBackendEnum",
attn_selector_config: "AttentionSelectorConfig",
num_heads: int | None = None,
) -> str:
return "vllm_add_dummy_platform.dummy_attention_backend.DummyAttentionBackend" # noqa E501
@@ -1,30 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.metrics.loggers import StatLoggerBase
class DummyStatLogger(StatLoggerBase):
"""A dummy stat logger for testing purposes.
Implements the minimal interface expected by StatLoggerManager.
"""
def __init__(self, vllm_config, engine_idx=0):
self.vllm_config = vllm_config
self.engine_idx = engine_idx
self.recorded = []
self.logged = False
self.engine_initialized = False
def record(
self, scheduler_stats, iteration_stats, mm_cache_stats=None, engine_idx=0
):
self.recorded.append(
(scheduler_stats, iteration_stats, mm_cache_stats, engine_idx)
)
def log(self):
self.logged = True
def log_engine_initialized(self):
self.engine_initialized = True
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from setuptools import setup
setup(
name="dummy_stat_logger",
version="0.1",
packages=["dummy_stat_logger"],
entry_points={
"vllm.stat_logger_plugins": [
"dummy_stat_logger = dummy_stat_logger.dummy_stat_logger:DummyStatLogger" # noqa
]
},
)
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.engine.arg_utils import EngineArgs
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.engine.llm_engine import LLMEngine
class DummyV1Scheduler(Scheduler):
def schedule(self, throttle_prefills: bool = False):
raise Exception("Exception raised by DummyV1Scheduler")
def test_scheduler_plugins_v1(monkeypatch: pytest.MonkeyPatch):
with monkeypatch.context() as m:
# Explicitly turn off engine multiprocessing so
# that the scheduler runs in this process
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with pytest.raises(Exception) as exception_info:
engine_args = EngineArgs(
model="facebook/opt-125m",
enforce_eager=True, # reduce test time
scheduler_cls=DummyV1Scheduler,
)
engine = LLMEngine.from_engine_args(engine_args=engine_args)
sampling_params = SamplingParams(max_tokens=1)
engine.add_request("0", "foo", sampling_params)
engine.step()
assert str(exception_info.value) == "Exception raised by DummyV1Scheduler"
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from dummy_stat_logger.dummy_stat_logger import DummyStatLogger
from vllm.config import VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.loggers import load_stat_logger_plugin_factories
def test_stat_logger_plugin_is_discovered(monkeypatch: pytest.MonkeyPatch):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "dummy_stat_logger")
factories = load_stat_logger_plugin_factories()
assert len(factories) == 1, f"Expected 1 factory, got {len(factories)}"
assert factories[0] is DummyStatLogger, (
f"Expected DummyStatLogger class, got {factories[0]}"
)
# instantiate and confirm the right type
vllm_config = VllmConfig()
instance = factories[0](vllm_config)
# The `is` check above narrows DummyStatLogger to a Callable
assert isinstance(instance, DummyStatLogger)
def test_no_plugins_loaded_if_env_empty(monkeypatch: pytest.MonkeyPatch):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "")
factories = load_stat_logger_plugin_factories()
assert factories == []
def test_invalid_stat_logger_plugin_raises(monkeypatch: pytest.MonkeyPatch):
def fake_plugin_loader(group: str):
assert group == "vllm.stat_logger_plugins"
return {"bad": object()}
with monkeypatch.context() as m:
m.setattr(
"vllm.v1.metrics.loggers.load_plugins_by_group",
fake_plugin_loader,
)
with pytest.raises(
TypeError,
match="Stat logger plugin 'bad' must be a subclass of StatLoggerBase",
):
load_stat_logger_plugin_factories()
@pytest.mark.asyncio
async def test_stat_logger_plugin_integration_with_engine(
monkeypatch: pytest.MonkeyPatch,
):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "dummy_stat_logger")
engine_args = AsyncEngineArgs(
model="facebook/opt-125m",
enforce_eager=True, # reduce test time
disable_log_stats=True, # disable default loggers
)
engine = AsyncLLM.from_engine_args(engine_args=engine_args)
assert engine.logger_manager is not None
assert len(engine.logger_manager.stat_loggers) == 2
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 # type: ignore[attr-defined]
assert isinstance(
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], # type: ignore[attr-defined]
DummyStatLogger,
)
engine.shutdown()
View File
-21
View File
@@ -1,21 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import envs
from vllm.platforms import current_platform
@pytest.fixture(autouse=True)
def skip_unsupported_flashinfer_sampler():
# CI runs this suite with FlashInfer both explicitly enabled and disabled.
if (
current_platform.is_cuda()
and envs.is_set("VLLM_USE_FLASHINFER_SAMPLER")
and envs.VLLM_USE_FLASHINFER_SAMPLER
and current_platform.num_compute_units(torch.accelerator.current_device_index())
<= 16
):
pytest.skip("FlashInfer top-k masking requires more than 16 SMs")
-39
View File
@@ -109,45 +109,6 @@ def _write_json(path: Path, value: object) -> None:
path.write_text(json.dumps(value), encoding="utf-8")
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
@pytest.mark.parametrize(
("is_mm_prefix_lm", "is_multimodal_model", "expected"),
[
pytest.param(True, True, True, id="multimodal-prefix-lm"),
pytest.param(False, True, False, id="multimodal-causal"),
pytest.param(True, False, False, id="text-prefix-lm"),
pytest.param(None, True, False, id="missing-model-config"),
],
)
def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
is_mm_prefix_lm: bool | None,
is_multimodal_model: bool,
expected: bool,
) -> None:
from vllm.platforms.rocm import RocmPlatform
config = SimpleNamespace(
compilation_config=SimpleNamespace(cudagraph_mode=CUDAGraphMode.NONE),
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
worker_cls="test-worker",
),
model_config=(
None
if is_mm_prefix_lm is None
else SimpleNamespace(is_mm_prefix_lm=is_mm_prefix_lm)
),
scheduler_config=SimpleNamespace(
is_multimodal_model=is_multimodal_model,
disable_chunked_mm_input=False,
),
)
RocmPlatform.check_and_update_config(config)
assert config.scheduler_config.disable_chunked_mm_input is expected
def _sampling_replay_config(
*,
return_sampling_mask: bool = True,
-4
View File
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE: Since CI runs the tests from the `tests` directory, it is necessary to rename
# this module to avoid conflicting with HF's `tokenizers` package
@@ -1,81 +0,0 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a specific location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name"
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "Temperature unit"
}
},
"required": ["location"]
}
}
},
{
"type": "function",
"function": {
"name": "search",
"description": "Search the web for information",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
},
"num_results": {
"type": "integer",
"description": "Number of results to return"
}
},
"required": ["query"]
}
}
}
],
"messages": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "What's the weather in Beijing?"
},
{
"role": "assistant",
"reasoning": "The user wants to know the weather in Beijing. I should use the get_weather tool.",
"tool_calls": [
{
"id": "call_001",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call_001",
"content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"
},
{
"role": "assistant",
"reasoning": "Got the weather data. Let me format a nice response.",
"content": "The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."
}
]
}
@@ -1,24 +0,0 @@
[
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Hello"
},
{
"role": "assistant",
"reasoning": "The user said hello, I should greet back.",
"content": "Hi there! How can I help you?"
},
{
"role": "user",
"content": "What is the capital of France?"
},
{
"role": "assistant",
"reasoning": "The user asks about the capital of France. It is Paris.",
"content": "The capital of France is Paris."
}
]
@@ -1,159 +0,0 @@
[
{
"role": "system",
"content": "该助手为DeepSeek,由深度求索公司创造。"
},
{
"role": "latest_reminder",
"content": "2026-02-21,星期六,广州,App,中文"
},
{
"role": "developer",
"content": "小柴胡冲剂和布洛芬能一起吃吗?\n\nCITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】",
"tools": [
{
"type": "function",
"function": {
"name": "search",
"description": "Web search. Split multiple queries with '||'.",
"parameters": {
"type": "object",
"properties": {
"queries": {
"type": "string",
"description": "query1||query2"
}
},
"required": [
"queries"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
{
"type": "function",
"function": {
"name": "open",
"description": "Batch open IDs (format 【{id}†...】) or URLs.",
"parameters": {
"type": "object",
"properties": {
"open_list": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"description": "ID or URL",
"anyOf": [
{
"type": "integer"
},
{
"type": "string"
}
],
"default": -1
},
"cursor": {
"type": "integer",
"description": "",
"default": -1
},
"loc": {
"type": "integer",
"description": "Start line",
"default": -1
},
"num_lines": {
"type": "integer",
"description": "",
"default": -1
},
"view_source": {
"type": "boolean",
"description": "",
"default": false
}
},
"additionalProperties": false
},
"description": ""
}
},
"required": [
"open_list"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
{
"type": "function",
"function": {
"name": "find",
"description": "Find exact text pattern in pages.",
"parameters": {
"type": "object",
"properties": {
"find_list": {
"type": "array",
"items": {
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": ""
},
"cursor": {
"type": "integer",
"description": "",
"default": -1
}
},
"required": [
"pattern"
],
"additionalProperties": false
},
"description": ""
}
},
"required": [
"find_list"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
}
]
},
{
"role": "assistant",
"content": "",
"reasoning": "用户想知道小柴胡冲剂和布洛芬能否一起服用。",
"tool_calls": [
{
"type": "function",
"function": {
"name": "search",
"arguments": "{\"queries\": \"小柴胡冲剂 布洛芬 相互作用 一起吃\"}"
}
}
]
},
{
"role": "tool",
"content": "[0]"
},
{
"role": "assistant",
"content": "请及时就医。",
"reasoning": "现在开始组织回答。",
"tool_calls": []
}
]
@@ -1,28 +0,0 @@
[
{
"role": "system",
"content": "该助手为DeepSeek-V3,由深度求索公司创造。\n今天是2025年10月17日,星期五。"
},
{
"role": "latest_reminder",
"content": "2024-11-15,上海市,App,中文"
},
{
"role": "user",
"content": "热海大滚锅是世界著名温泉吗"
},
{
"role": "assistant",
"content": "关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。\n\n为了让你快速了解,我整理了一个简要的表格:\n\n| 维度 | 热海大滚锅的地位与特点 |\n| :--- | :--- |\n| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |\n| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |\n| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |\n| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |\n\n### 💡 游玩攻略与温馨提示\n\n如果你计划前往热海大滚锅,这里有一些实用信息供你参考:\n\n- **门票与开放时间**:\n - **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。\n - **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。\n\n- **特色体验**:\n - **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。\n - **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。\n\n- **注意事项**:\n - **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。\n - **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。\n\n希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。",
"mask": 1
},
{
"role": "user",
"content": "世界著名温泉有哪些",
"task": "action"
},
{
"role": "assistant",
"content": "Search"
}
]
@@ -1,39 +0,0 @@
{
"thinking_mode": "thinking",
"reasoning_effort": "max",
"messages": [
{
"role": "system",
"content": "You are a helpful vision assistant."
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "请按“第一张、第二张”的顺序回答:第一张图"
},
{
"type": "image_url",
"image_url": {
"url": "examples/images/carrots.jpeg"
}
},
{
"type": "text",
"text": "和第二张图"
},
{
"type": "image_url",
"image_url": {
"url": "examples/images/corn.jpeg"
}
},
{
"type": "text",
"text": "中分别是什么食材?它们通常食用的部位分别是什么?"
}
]
}
]
}
@@ -1,36 +0,0 @@
<|begin▁of▁sentence|>You are a helpful assistant.
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
<|DSML|tool_calls>
<|DSML|invoke name="$TOOL_NAME">
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
...
</|DSML|invoke>
<|DSML|invoke name="$TOOL_NAME2">
...
</|DSML|invoke>
</|DSML|tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
<|DSML|tool_calls>
<|DSML|invoke name="get_weather">
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
<|DSML|parameter name="unit" string="true">celsius</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
@@ -1 +0,0 @@
<|begin▁of▁sentence|>You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
@@ -1,38 +0,0 @@
<|begin▁of▁sentence|>该助手为DeepSeek,由深度求索公司创造。<|latest_reminder|>2026-02-21,星期六,广州,App,中文<|User|>小柴胡冲剂和布洛芬能一起吃吗?
CITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
<|DSML|tool_calls>
<|DSML|invoke name="$TOOL_NAME">
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
...
</|DSML|invoke>
<|DSML|invoke name="$TOOL_NAME2">
...
</|DSML|invoke>
</|DSML|tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "search", "description": "Web search. Split multiple queries with '||'.", "parameters": {"type": "object", "properties": {"queries": {"type": "string", "description": "query1||query2"}}, "required": ["queries"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
{"name": "open", "description": "Batch open IDs (format 【{id}†...】) or URLs.", "parameters": {"type": "object", "properties": {"open_list": {"type": "array", "items": {"type": "object", "properties": {"id": {"description": "ID or URL", "anyOf": [{"type": "integer"}, {"type": "string"}], "default": -1}, "cursor": {"type": "integer", "description": "", "default": -1}, "loc": {"type": "integer", "description": "Start line", "default": -1}, "num_lines": {"type": "integer", "description": "", "default": -1}, "view_source": {"type": "boolean", "description": "", "default": false}}, "additionalProperties": false}, "description": ""}}, "required": ["open_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
{"name": "find", "description": "Find exact text pattern in pages.", "parameters": {"type": "object", "properties": {"find_list": {"type": "array", "items": {"type": "object", "properties": {"pattern": {"type": "string", "description": ""}, "cursor": {"type": "integer", "description": "", "default": -1}}, "required": ["pattern"], "additionalProperties": false}, "description": ""}}, "required": ["find_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|Assistant|><think>用户想知道小柴胡冲剂和布洛芬能否一起服用。</think>
<|DSML|tool_calls>
<|DSML|invoke name="search">
<|DSML|parameter name="queries" string="true">小柴胡冲剂 布洛芬 相互作用 一起吃</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>[0]</tool_result><|Assistant|><think>现在开始组织回答。</think>请及时就医。<|end▁of▁sentence|>
@@ -1,29 +0,0 @@
<|begin▁of▁sentence|>该助手为DeepSeek-V3,由深度求索公司创造。
今天是2025年10月17日,星期五。<|latest_reminder|>2024-11-15,上海市,App,中文<|User|>热海大滚锅是世界著名温泉吗<|Assistant|></think>关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。
为了让你快速了解,我整理了一个简要的表格:
| 维度 | 热海大滚锅的地位与特点 |
| :--- | :--- |
| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |
| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |
| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |
| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |
### 💡 游玩攻略与温馨提示
如果你计划前往热海大滚锅,这里有一些实用信息供你参考:
- **门票与开放时间**:
- **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。
- **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。
- **特色体验**:
- **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。
- **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。
- **注意事项**:
- **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。
- **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。
希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。<|end▁of▁sentence|><|User|>世界著名温泉有哪些<|Assistant|></think><|action|>Search<|end▁of▁sentence|>
@@ -1,9 +0,0 @@
<|begin▁of▁sentence|>You are a helpful vision assistant.<|User|>请按“第一张、第二张”的顺序回答:第一张图
<|deepseek_image|>
和第二张图
<|deepseek_image|>
中分别是什么食材?它们通常食用的部位分别是什么?<|Assistant|></think>
@@ -1,38 +0,0 @@
<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)
You are a helpful assistant.
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
<|DSML| calls>
<|DSML| invoke name="$TOOL_NAME">
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
...
</|DSML| invoke>
<|DSML| invoke name="$TOOL_NAME2">
...
</|DSML| invoke>
</|DSML| calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
<|DSML| calls>
<|DSML| invoke name="get_weather">
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
</|DSML| invoke>
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
@@ -1,3 +0,0 @@
<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.config import (
CacheConfig,
ModelConfig,
ParallelConfig,
SchedulerConfig,
VllmConfig,
)
from vllm.sampling_params import SamplingParams
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
from vllm.v1.structured_output import StructuredOutputManager
BLOCK_SIZE = 16
NUM_BLOCKS = 6
MAX_MODEL_LEN = 1000
LONG_PREFILL_THRESHOLD = 15
EOS_TOKEN_ID = 50256
def build_scheduler():
model_config = ModelConfig(
model="facebook/opt-350m",
trust_remote_code=True,
dtype="float16",
seed=42,
skip_tokenizer_init=True,
max_model_len=MAX_MODEL_LEN,
)
scheduler_config = SchedulerConfig(
max_num_seqs=5,
max_num_batched_tokens=200,
max_model_len=MAX_MODEL_LEN,
long_prefill_token_threshold=LONG_PREFILL_THRESHOLD,
enable_chunked_prefill=True,
is_encoder_decoder=model_config.is_encoder_decoder,
policy="priority",
watermark=0.0,
scheduler_reserve_full_isl=False,
)
cache_config = CacheConfig(
block_size=BLOCK_SIZE,
gpu_memory_utilization=0.9,
cache_dtype="auto",
enable_prefix_caching=False,
)
vllm_config = VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
cache_config=cache_config,
parallel_config=ParallelConfig(),
)
kv_cache_spec = FullAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=1, dtype=torch.float32
)
kv_cache_config = KVCacheConfig(
num_blocks=NUM_BLOCKS,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
cache_config.num_gpu_blocks = NUM_BLOCKS
register_all_kvcache_specs(vllm_config)
return Scheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
block_size=BLOCK_SIZE,
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
)
_hash_init = False
def make_request(req_id, num_tokens, priority, arrival_time):
global _hash_init
if not _hash_init:
init_none_hash(sha256)
_hash_init = True
block_hasher = get_request_block_hasher(BLOCK_SIZE, sha256)
sp = SamplingParams(ignore_eos=True, max_tokens=50)
sp.update_from_generation_config({}, EOS_TOKEN_ID)
return Request(
request_id=req_id,
prompt_token_ids=[7] * num_tokens,
sampling_params=sp,
pooling_params=None,
priority=priority,
arrival_time=arrival_time,
block_hasher=block_hasher,
)
def mock_output(out, tok=100):
ids = list(out.num_scheduled_tokens.keys())
return ModelRunnerOutput(
req_ids=ids,
req_id_to_index={r: i for i, r in enumerate(ids)},
sampled_token_ids=[[tok] for _ in ids],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
)
def test_priority_scheduler_preempt_skipped_request():
scheduler = build_scheduler()
# R1: Add request A (Worst priority)
A = make_request("A", num_tokens=200, priority=9, arrival_time=1.0)
scheduler.add_request(A)
out = scheduler.schedule()
scheduler.update_from_output(out, mock_output(out))
# R2: Add request B (Best priority)
B = make_request("B", num_tokens=15, priority=0, arrival_time=2.0)
scheduler.add_request(B)
out = scheduler.schedule()
scheduler.update_from_output(out, mock_output(out))
# R3: Add request C (Middle priority)
C = make_request("C", num_tokens=1, priority=1, arrival_time=3.0)
scheduler.add_request(C)
out = scheduler.schedule()
scheduler.update_from_output(out, mock_output(out))
# R4: Trigger preemption
out = scheduler.schedule(throttle_prefills=True)
is_c_scheduled = "C" in out.num_scheduled_tokens
assert is_c_scheduled, (
"Bug present: Request C was silently skipped during scheduling "
"because the req_index was not decremented after preempting A."
)
assert C.status == RequestStatus.RUNNING
-138
View File
@@ -4,7 +4,6 @@ import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
@@ -13,7 +12,6 @@ import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
@@ -25,7 +23,6 @@ if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
@@ -185,7 +182,6 @@ class ExternalLBServerManager:
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
@@ -299,140 +295,6 @@ async def clients(servers: list[tuple[RemoteOpenAIServer, list[str]]]):
]
def _get_parallel_config(server: RemoteOpenAIServer):
response = requests.get(server.url_for("server_info?config_format=json"))
response.raise_for_status()
vllm_config = response.json()["vllm_config"]
return vllm_config["parallel_config"]
def _assert_expert_parallel_config(
servers: list[tuple[RemoteOpenAIServer, list[str]]],
):
for server, server_args in servers:
expected = "--enable-expert-parallel" in server_args
parallel_config = _get_parallel_config(server)
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")
# Each request will hit one of the API servers
# `n_reqs` is set so that there is a good chance each server
# receives at least one request
n_reqs = 2 * api_server_count * api_server_count
parallel_configs = [_get_parallel_config(server) for _ in range(n_reqs)]
api_process_counts = [c["_api_process_count"] for c in parallel_configs]
api_process_ranks = [c["_api_process_rank"] for c in parallel_configs]
assert all(c == api_server_count for c in api_process_counts), (
api_process_counts
)
assert all(0 <= r < api_server_count for r in api_process_ranks), (
api_process_ranks
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,195 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import AutoTokenizer
from vllm.sampling_params import SamplingParams
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
# ruff: noqa: E501
def test_fast_inc_detok_invalid_utf8_err_case():
"""Test edge case where tokenizer can produce non-monotonic,
invalid UTF-8 output, which breaks the internal state of
tokenizers' DecodeStream.
See https://github.com/vllm-project/vllm/issues/17448.
Thanks to reproducer from @fpaupier:
https://gist.github.com/fpaupier/0ed1375bd7633c5be6c894b1c7ac1be3.
"""
tokenizer = AutoTokenizer.from_pretrained("google/gemma-3-1b-it")
# Create a test request
prompt_token_ids = [107, 4606, 236787, 107]
params = SamplingParams(skip_special_tokens=True)
request = EngineCoreRequest(
request_id="test",
external_req_id="test-ext",
prompt_token_ids=prompt_token_ids,
mm_features=None,
sampling_params=params,
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
detokenizer = IncrementalDetokenizer.from_new_request(tokenizer, request)
assert detokenizer.__class__.__name__ == "FastIncrementalDetokenizer", (
"Should use FastIncrementalDetokenizer by default"
)
# Process tokens incrementally
test_tokens = [
236840,
107,
138,
236782,
107,
140,
236775,
6265,
1083,
623,
121908,
147418,
827,
107,
140,
236775,
6265,
236779,
2084,
1083,
623,
203292,
827,
107,
140,
236775,
6265,
236779,
7777,
1083,
623,
121908,
147418,
569,
537,
236789,
65880,
569,
537,
236789,
62580,
853,
115693,
210118,
35178,
16055,
1270,
759,
215817,
4758,
1925,
1117,
827,
107,
140,
236775,
5654,
1083,
623,
110733,
46291,
827,
107,
140,
236775,
5654,
236779,
2084,
1083,
623,
136955,
56731,
827,
107,
140,
236775,
5654,
236779,
7777,
1083,
623,
194776,
2947,
496,
109811,
1608,
890,
215817,
4758,
1925,
1117,
2789,
432,
398,
602,
31118,
569,
124866,
134772,
509,
19478,
1640,
33779,
236743,
236770,
236819,
236825,
236771,
432,
398,
432,
237167,
827,
107,
140,
236775,
77984,
1083,
623,
2709,
236745,
2555,
513,
236789,
602,
31118,
569,
]
output = ""
for i, token_id in enumerate(test_tokens):
detokenizer.update([token_id], False)
finished = i == len(test_tokens) - 1
output += detokenizer.get_next_output_text(finished, delta=True)
assert (
output
== r"""[
{
"source": "Résultats",
"source_type": "CONCEPT",
"source_description": "Résultats de l'analyse de l'impact des opérations israéliennes sur la frontière libanaise",
"target": "Israël",
"target_type": "ORGANIZATION",
"target_description": "Pays qui a obtenu à sa frontière libanaise « un niveau de calme inédit depuis les années 1960 »",
"relationship": "Obtention d'un niveau de"""
)
+552
View File
@@ -0,0 +1,552 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from itertools import chain, count
from typing import Any, Literal
from unittest.mock import Mock
import numpy as np
import torch
from vllm import SamplingParams
from vllm.config import (
AttentionConfig,
CacheConfig,
DeviceConfig,
KVTransferConfig,
ModelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
ExampleConnector,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
from vllm.v1.core.sched.scheduler import Scheduler, SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
EOS_TOKEN_ID = 50256
def create_mock_connector() -> Mock:
connector = Mock()
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
)
return connector
def maybe_update_block_size(block_size: int) -> int:
return 64 if current_platform.is_xpu() else block_size
def assert_scheduler_empty(scheduler: Scheduler):
"""Confirm the scheduler is "empty" - i.e. no leaks."""
# Scheduler Metadata.
assert len(scheduler.requests) == 0
assert len(scheduler.waiting) == 0
assert len(scheduler.kv_holding_waiting) == 0
assert not scheduler.deferred_waiting
assert len(scheduler.running) == 0
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
assert len(scheduler._inflight_prefills) == 0
assert not scheduler._kv_fetch_stages
assert not any(scheduler._kv_fetch_counts.values())
# EncoderCacheManager.
assert len(scheduler.encoder_cache_manager.freed) == 0
assert len(scheduler.encoder_cache_manager.cached) == 0
# KVCache Manager.
assert (
len(
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks
)
== 0
)
assert (
len(
scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].num_cached_block
)
== 0
)
num_free_blocks = (
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
)
assert num_free_blocks == (scheduler.kv_cache_manager.block_pool.num_gpu_blocks - 1)
# NOTE(rob): just the ref count on blocks will be 0. The hash
# value, etc will remain since we lazily evict for prefix cache.
for block in scheduler.kv_cache_manager.block_pool.blocks:
assert block.ref_cnt == 0
def create_vllm_config(
model: str = "facebook/opt-125m",
max_num_seqs: int = 16,
max_num_batched_tokens: int = 64,
block_size: int = 16,
max_model_len: int = 10000,
enable_chunked_prefill: bool = True,
enable_permute_local_kv: bool = False,
kv_connector_extra_config: dict[str, Any] | None = None,
dtype: str = "float16",
cache_dtype: str = "auto",
hf_overrides: dict[str, Any] | None = None,
attention_backend: str | None = None,
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
kv_connector: str = "NixlConnector",
kv_connector_module_path: str | None = None,
kv_role: str = "kv_consumer",
disable_hybrid_kv_cache_manager: bool | None = None,
num_speculative_tokens: int | None = None,
) -> VllmConfig:
"""Initialize VllmConfig For Testing."""
model_config = ModelConfig(
model=model,
trust_remote_code=True,
dtype=dtype,
seed=42,
hf_overrides=hf_overrides or {},
)
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
enable_chunked_prefill=enable_chunked_prefill,
is_encoder_decoder=model_config.is_encoder_decoder,
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
)
# Cache config, optionally force APC
cache_config = CacheConfig(
block_size=block_size,
gpu_memory_utilization=0.9,
cache_dtype=cache_dtype,
enable_prefix_caching=True,
)
# Connectors are constructed after layout resolution; mirror that here.
cache_config.kv_cache_layout = "LBNHC"
kv_transfer_config = KVTransferConfig(
kv_connector=kv_connector,
kv_connector_module_path=kv_connector_module_path,
kv_role=kv_role,
enable_permute_local_kv=enable_permute_local_kv,
kv_connector_extra_config=kv_connector_extra_config or {},
kv_load_failure_policy=kv_load_failure_policy,
)
attention_config = AttentionConfig(backend=attention_backend)
speculative_config = (
SpeculativeConfig(model="ngram", num_speculative_tokens=num_speculative_tokens)
if num_speculative_tokens is not None
else None
)
return VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
cache_config=cache_config,
kv_transfer_config=kv_transfer_config,
device_config=DeviceConfig("cpu"),
attention_config=attention_config,
speculative_config=speculative_config,
)
def create_scheduler(
vllm_config: VllmConfig,
num_blocks: int = 10000,
kv_cache_config: KVCacheConfig | None = None,
hash_block_size: int | None = None,
) -> Scheduler | AsyncScheduler:
"""Initialize Scheduler For Testing."""
block_size = vllm_config.cache_config.block_size
if kv_cache_config is None:
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks, # A large number of blocks to hold all requests
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
],
)
vllm_config.cache_config.num_gpu_blocks = num_blocks
scheduler_cls = (
AsyncScheduler if vllm_config.scheduler_config.async_scheduling else Scheduler
)
return scheduler_cls(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=hash_block_size,
)
_request_count = count(1)
_none_hash_initialized = False
def create_request(
request_id: int | None = None,
num_tokens: int = 10,
common_prefix_len=0,
max_tokens: int = 16,
do_remote_decode: bool = False,
do_remote_prefill: bool = False,
num_remote_blocks: int = 3,
block_size: int = 16,
hash_fn: Callable = sha256,
) -> Request:
"""Make dummy request for testing."""
assert num_tokens >= common_prefix_len >= 0
if request_id is None:
request_id = next(_request_count)
global _none_hash_initialized
if not _none_hash_initialized:
init_none_hash(hash_fn)
_none_hash_initialized = True
kv_transfer_params: dict[str, Any] | None = None
if do_remote_decode:
assert not do_remote_prefill
kv_transfer_params = dict(do_remote_prefill=False, do_remote_decode=True)
elif do_remote_prefill:
kv_transfer_params = dict(
do_remote_prefill=True,
do_remote_decode=False,
remote_engine_id="my-engine-id",
remote_request_id=f"prefill-{request_id}",
remote_block_ids=list(range(num_remote_blocks)),
remote_host="my-host",
remote_port=1234,
tp_size=1,
)
max_tokens = 1 if do_remote_decode else max_tokens
sampling_params = SamplingParams(max_tokens=max_tokens)
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
common_prefix = [1] * common_prefix_len if common_prefix_len > 0 else []
suffix = [i * request_id for i in range(num_tokens - common_prefix_len)]
prompt_token_ids = common_prefix + suffix
req = Request(
request_id=f"id-{request_id}",
prompt_token_ids=prompt_token_ids,
sampling_params=sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=get_request_block_hasher(block_size, hash_fn),
)
req.kv_transfer_params = kv_transfer_params
return req
def create_model_runner_output(
reqs: list[Request],
finished_sending: set[str] | None = None,
finished_recving: set[str] | None = None,
invalid_block_ids: set[int] | None = None,
use_eos: bool = False,
token_id: int = 0,
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
) -> ModelRunnerOutput:
"""Make dummy model runner output for testing."""
# Make request data.
req_ids = [req.request_id for req in reqs]
req_id_to_index = {req_id: idx for idx, req_id in enumerate(req_ids)}
# Make sampled tokens.
sampled_token = EOS_TOKEN_ID if use_eos else token_id
sampled_token_ids = [[sampled_token] for _ in req_ids]
kv_connector_output = (
None
if (
finished_sending is None
and finished_recving is None
and invalid_block_ids is None
and kv_connector_worker_meta is None
)
else KVConnectorOutput(
finished_sending=finished_sending,
finished_recving=finished_recving,
invalid_block_ids=invalid_block_ids or set(),
kv_connector_worker_meta=kv_connector_worker_meta,
)
)
# Make output data structure.
return ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index=req_id_to_index,
sampled_token_ids=sampled_token_ids,
logprobs=None,
prompt_logprobs_dict={},
pooler_output=None,
kv_connector_output=kv_connector_output,
)
class TestExampleConnector(ExampleConnector):
def __init__(
self,
config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
):
self.name = config.kv_transfer_config.kv_connector_extra_config["name"]
self._connector = ExampleConnector(config, role, kv_cache_config)
self.call_record: dict[str, int] = defaultdict(int)
# Use a unique temp file per connector
self._event_file = (
tempfile.gettempdir()
+ f"/connector_{self.name}-{self.role.name}_events.log"
)
# Start with an empty file
with open(self._event_file, "w") as _:
pass
def __getattribute__(self, name):
if name in (
"_connector",
"call_record",
"name",
"_event_file",
"__class__",
"__dict__",
"__getattribute__",
"__init__",
): # avoid recursion
return object.__getattribute__(self, name)
if not hasattr(self._connector, name):
return object.__getattribute__(self, name)
attr = getattr(self._connector, name)
# Intercept calls to the connector interface and write an event
# for each one to a file, which can be read back in the main test proc.
if callable(attr):
def wrapper(*args, **kwargs):
self.call_record[name] += 1
# Include args that we're interested in
to_log = [name]
for arg in args:
if isinstance(arg, int):
to_log.append(str(arg))
elif isinstance(arg, KVCacheBlocks):
to_log.append(f"num_blocks={[len(b) for b in arg.blocks]}")
# Log the event as a line to the file
try:
with open(self._event_file, "a") as f:
f.write(" ".join(to_log) + "\n")
except Exception as e:
print(f"[ERROR] Could not log event {name} for {self.name}: {e}")
return attr(*args, **kwargs)
return wrapper
return attr
@dataclass(frozen=True)
class MockKVConfig:
matched_tokens: int = 0
is_async: bool = False
num_defers_before_matching: int = 0
supports_divergent_local_hybrid_hits: bool = False
class MockKVConnectorMetadata(KVConnectorMetadata):
def __init__(self):
# Scheduler tests check metadata.requests
self.requests: list = []
class MockKVConnector(KVConnectorBase_V1):
"""Mock KV connector for scheduler tests, supporting both sync and async mode."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
):
super().__init__(vllm_config, role, kv_cache_config)
extra_config = self._kv_transfer_config.kv_connector_extra_config
self.config = MockKVConfig(
matched_tokens=extra_config["matched_tokens"],
is_async=extra_config["is_async"],
num_defers_before_matching=extra_config.get(
"num_defers_before_matching", 0
),
supports_divergent_local_hybrid_hits=extra_config.get(
"supports_divergent_local_hybrid_hits", False
),
)
self._defers_left: defaultdict[str, int] = defaultdict(
lambda: self.config.num_defers_before_matching
)
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return self.config.supports_divergent_local_hybrid_hits
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
if self._defers_left[request.request_id] > 0:
self._defers_left[request.request_id] -= 1
return (None, False)
return (self.config.matched_tokens, self.config.is_async)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
pass
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
metadata = MockKVConnectorMetadata()
cached_reqs = scheduler_output.scheduled_cached_reqs
for req_id in chain(
(req.req_id for req in scheduler_output.scheduled_new_reqs),
(
req_id
for req_id in cached_reqs.req_ids
if req_id in cached_reqs.resumed_req_ids
),
):
metadata.requests.append({"req_id": req_id})
return metadata
def start_load_kv(self, kv_caches, finished_req_ids):
pass
def wait_for_layer_load(self, layer_name):
pass
def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs):
pass
def wait_for_save(self):
pass
KVConnectorFactory.register_connector(
"TestExampleConnector", __name__, TestExampleConnector.__name__
)
KVConnectorFactory.register_connector(
"MockKVConnector", __name__, MockKVConnector.__name__
)
def make_kv_cache_config(
block_size: int,
swa_enabled: bool = False,
mamba_enabled: bool = False,
sw_size: int = 128,
num_blocks: int = 100,
mamba_cache_mode: Literal["align", "none"] = "none",
) -> KVCacheConfig:
kv_cache_groups = [
KVCacheGroupSpec(
["layer0", "layer2"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
)
]
if swa_enabled:
kv_cache_groups.append(
KVCacheGroupSpec(
["layer1", "layer3"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
sliding_window=sw_size,
),
)
)
if mamba_enabled:
kv_cache_groups.append(
KVCacheGroupSpec(
["mamba0", "mamba1"],
MambaSpec(
block_size=block_size,
shapes=((16,), (16,)),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
),
)
)
return KVCacheConfig(
num_blocks=num_blocks, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
by the connector can be resolved to (addr, len, dev) byte ranges.
"""
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
counts = strided_descs[:, 4].astype(np.intp)
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
out = np.repeat(strided_descs[:, :3], counts, axis=0)
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
return out
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501
DummyStatLogger,
)
from tests.utils import wait_for_memory_to_settle
from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM
from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger
@pytest.fixture
def log_stats_enabled_engine_args():
"""Shared fixture providing common AsyncEngineArgs configuration
used across multiple tests.
"""
return AsyncEngineArgs(
model="distilbert/distilgpt2",
dtype="half",
disable_log_stats=False,
enforce_eager=True,
)
@pytest.mark.asyncio
async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args):
"""RayPrometheusStatLogger should replace the default PrometheusStatLogger."""
engine = AsyncLLM.from_engine_args(
log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger]
)
try:
assert isinstance(
engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger
)
finally:
engine.shutdown()
wait_for_memory_to_settle(
threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization
)
@pytest.mark.asyncio
async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args):
"""It's still possible to use custom stat loggers exclusively by passing
disable_log_stats=True in addition to a list of custom stat loggers.
"""
# Create engine_args with disable_log_stats=True for this test
disabled_log_engine_args = copy.deepcopy(log_stats_enabled_engine_args)
disabled_log_engine_args.disable_log_stats = True
# Disable default loggers; pass custom stat logger to the constructor
engine = AsyncLLM.from_engine_args(
disabled_log_engine_args, stat_loggers=[DummyStatLogger]
)
try:
assert len(engine.logger_manager.stat_loggers) == 2
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
assert isinstance(
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
DummyStatLogger,
)
# log_stats is still True, since custom stat loggers are used
assert engine.log_stats
finally:
engine.shutdown()
wait_for_memory_to_settle(
threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization
)
View File
-35
View File
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable
from pathlib import Path
import pytest
from vllm.platforms import current_platform
@pytest.fixture
def spawn_sitecustomize_factory(monkeypatch, tmp_path: Path):
"""Return a function that installs a given sitecustomize payload.
Both ROCm and XPU force the `spawn` multiprocessing start method (see
`vllm.platforms.rocm`/`vllm.platforms.xpu`), which re-imports modules in
child processes instead of inheriting them via `fork`'s copy-on-write
memory. A plain `monkeypatch.setattr` in the parent test process is
therefore invisible to spawned EngineCore/worker subprocesses on these
platforms, so the payload must be installed via `sitecustomize.py` on
`PYTHONPATH` instead.
"""
if not (current_platform.is_rocm() or current_platform.is_xpu()):
return lambda _: None
def install(lines: Iterable[str]) -> None:
sc = tmp_path / "sitecustomize.py"
sc.write_text("\n".join(lines) + "\n")
monkeypatch.setenv(
"PYTHONPATH",
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
)
return install
-6
View File
@@ -1,6 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shutdown test utils."""
SHUTDOWN_TEST_TIMEOUT_SEC = 120
SHUTDOWN_TEST_THRESHOLD_BYTES = 2 * 2**30
@@ -1,611 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import ClassVar
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
from vllm.v1.structured_output.backend_xgrammar import (
XgrammarBackend,
has_xgrammar_unsupported_json_features,
validate_xgrammar_grammar,
)
from vllm.v1.structured_output.utils import choice_as_grammar
pytestmark = pytest.mark.cpu_test
def grammar_accepts(schema: dict, text: str) -> bool:
return _is_grammar_accept_string(Grammar.from_json_schema(schema), text)
# ================================================
# Unsupported schemas
# ================================================
@pytest.fixture
def unsupported_multipleOf_schemas():
return [
{"type": "number", "multipleOf": 120},
{"Even": {"type": "number", "multipleOf": 2}},
{"type": "integer", "multipleOf": 120},
{"Even": {"type": "integer", "multipleOf": 2}},
]
@pytest.fixture
def unsupported_array_schemas():
return [
# array + some constraints is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968
{"type": "array", "uniqueItems": True},
{"type": "array", "contains": {"type": "string"}},
{"type": "array", "minContains": 1},
{"type": "array", "maxContains": 5},
]
@pytest.fixture
def unsupported_string_schemas():
return [
# ========================================================
# string + format is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967
{"type": "string", "format": "non_existing_format"},
# ========================================================
#
# ========================================================
# string + format/pattern + length constraint is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966
#
# pattern/format is compiled but length bounds are silently dropped,
# so the combination must be rejected instead of producing quietly
# wrong output
{"type": "string", "pattern": "^a+$", "maxLength": 2},
{"type": "string", "pattern": "^a+$", "minLength": 3},
{"type": "string", "format": "email", "maxLength": 10},
# ========================================================
]
@pytest.fixture
def unsupported_propertyNames_combinations():
return [
# ========================================================
# propertyNames + patternProperties is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959
#
# xgrammar drops propertyNames whenever patternProperties is present,
# so "grade_12" matches the pattern and escapes the name constraint
{
"type": "object",
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
},
# ========================================================
#
# ========================================================
# propertyNames + maxLength is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960
{
"type": "object",
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
},
# ========================================================
#
# ========================================================
# propertyNames + properties is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958
{
"type": "object",
"propertyNames": {"pattern": "^[a-z]+$"},
"properties": {"Bad": {"type": "integer"}},
"required": ["Bad"],
},
{
"type": "object",
"propertyNames": {"enum": ["good"]},
"properties": {"Bad": {"type": "integer"}},
},
# ========================================================
#
# ========================================================
# propertyNames + unevaluatedProperties is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961
#
# xgrammar's propertyNames-only branch falls back to an unconstrained
# value type, so it drops unevaluatedProperties whether it restricts
# values (a schema) or forbids them (false).
{
"type": "object",
"propertyNames": {"pattern": "^[a-z]+$"},
"unevaluatedProperties": False,
},
# also keep in mind that unevaluatedProperties can be a schema
{
"type": "object",
"propertyNames": {"pattern": "^[a-z]+$"},
"unevaluatedProperties": {"type": "integer"},
},
# ========================================================
]
@pytest.fixture
def unsupported_patternProperties_combinations():
return [
# ========================================================
# patternProperties + properties is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964
#
# JSON Schema requires a property matched by both `properties` and
# `patternProperties` to satisfy both (conjunction), but xgrammar
# compiles them as alternative branches, so satisfying either one is
# enough.
{
"type": "object",
"patternProperties": {"^x$": {"type": "integer"}},
"properties": {"x": {"type": "string"}},
},
# ========================================================
#
# ========================================================
# patternProperties + patternProperties is unsupported by xgrammar
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965
#
# The same alternative-branches problem applies to overlapping
# patternProperties patterns: a key matching both patterns only has to
# satisfy one of them.
{
"type": "object",
"patternProperties": {
"^a[a-z]*$": {"type": "string"},
"^[a-z]*z$": {"type": "integer"},
},
},
# ========================================================
]
@pytest.fixture
def unsupported_multibranch_allof():
"""Regression tests for issue #56556
[Bug]: JSON schema with multiple allOf branches is silently ignored
by the xgrammar structured-output backend #56556
Xgrammar currently does NOT support multi-branch allOf.
Bug reported to xgrammar team for tracking progress on resolving:
https://github.com/mlc-ai/xgrammar/issues/937
"""
return [
{
"allOf": [
{"type": "string"},
{"enum": ["yes", "no"]},
]
},
# Multi-branch test case with > 2 branches:
{
"allOf": [
{"type": "string"},
{"minLength": 2},
{"maxLength": 10},
]
},
{
"allOf": [
{"type": "integer"},
{"minimum": 0},
]
},
# Non root multi-branch allOf:
{
"type": "object",
"properties": {
"is_this_sparta": {
"allOf": [
{"type": "string"},
{"enum": ["yes", "no"]},
],
},
},
},
]
@pytest.fixture
def unsupported_vllm_issue_56556_schema():
"""Bug repro from vLLM issue #56556"""
return [
{
"$defs": {
"Base": {
"type": "object",
"properties": {"x": {"type": "integer", "minimum": 10}},
"required": ["x"],
}
},
"allOf": [
{"$ref": "#/$defs/Base"},
{
"type": "object",
"properties": {"y": {"type": "string"}},
"required": ["y"],
},
],
},
]
# ================================================
# Supported schemas
# ================================================
@pytest.fixture
def supported_frankenstein_schema():
# IMPORTANT(arpera):
# Do NOT add more keywords here! This schema is overcrowded enough that a new
# keyword can end up having no effect on the compiled grammar while the
# test still passes. Give a new keyword its own fixture and named test.
return {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"},
"email": {"type": "string", "format": "email"},
"status": {"type": "string"},
"scores": {"type": "array", "items": {"type": "number"}},
"car_type": {"type": "string", "enum": ["sedan", "suv", "truck"]},
"car_brand": {"type": "string", "pattern": "^[a-zA-Z]+$"},
"short_description": {"type": "string", "maxLength": 50},
"mileage": {"type": "number", "minimum": 0, "maximum": 1000000},
"model_year": {
"type": "integer",
"exclusiveMinimum": 1900,
"exclusiveMaximum": 2100,
},
"long_description": {"type": "string", "minLength": 50, "maxLength": 2000},
"address": {
"type": "object",
"properties": {
"street": {"type": "string"},
"city": {"type": "string"},
},
},
},
"minProperties": 1,
"maxProperties": 100,
}
@pytest.fixture
def pattern_properties_schema():
return {
"type": "object",
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
}
@pytest.fixture
def property_names_schema():
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
@pytest.fixture
def property_names_with_additional_properties_schema():
"""This combination propertyNames + additionalProperties
was unsupported some time ago by xgrammar.
So, this test case was in unsupported category before.
But this xgrammar's PR
https://github.com/mlc-ai/xgrammar/pull/836
added support of propertyNames + additionalProperties
This fix was released in xgrammar v0.2.7
which vLLM pins in PR #57272
"""
return {
"type": "object",
"propertyNames": {"pattern": "^[a-z_]+$"},
"additionalProperties": {"type": "integer"},
}
@pytest.fixture
def supported_allof_anyof_and_oneof():
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
This test case was made during working on covering
unsupported multi-branch allOf to make sure the bug
is NOT present in other combinators.
See for more context in this file test `unsupported_multibranch_allof`
"""
return [
# single-branch allOf:
{
"allOf": [
{"type": "string"},
],
},
# Corner cases: empty allOf, this is a valid schema
{"allOf": []},
# multi-branch allOf:
# is NOT supported yet, see vLLM issue #56556
# single-branch anyOf:
{
"anyOf": [],
},
# multi-branch anyOf:
{
"anyOf": [
{"type": "string"},
{"type": "integer"},
],
},
# single-branch oneOf:
{
"oneOf": [
{"type": "integer", "minimum": 0, "maximum": 100},
],
},
# multi-branch oneOf:
{
"oneOf": [
{"type": "integer", "minimum": 0, "maximum": 100},
{"type": "string", "enum": ["auto", "none"]},
],
},
# Corner case:
# "allOf" is a property name, not allOf combinator keyword
{"type": "object", "properties": {"allOf": {"type": "string"}}},
]
# ================================================
# Test has_xgrammar_unsupported_json_features functionality
# ================================================
class TestHasXGrammarUnsupportedJsonFeatures:
@pytest.mark.parametrize(
"schema_type",
[
"unsupported_string_schemas",
"unsupported_multipleOf_schemas",
"unsupported_array_schemas",
"unsupported_propertyNames_combinations",
"unsupported_patternProperties_combinations",
# vLLM issue #56556
"unsupported_multibranch_allof",
"unsupported_vllm_issue_56556_schema",
],
)
def test_unsupported_json_features(self, schema_type, request):
schemas = request.getfixturevalue(schema_type)
for schema in schemas:
assert has_xgrammar_unsupported_json_features(schema), (
f"Schema should be unsupported: {schema}"
)
@pytest.mark.parametrize(
"schema_type",
[
"supported_frankenstein_schema",
"pattern_properties_schema",
"property_names_schema",
"property_names_with_additional_properties_schema",
# Additional test cases implemented during work on
# vLLM issue #56556
"supported_allof_anyof_and_oneof",
],
)
def test_supported_json_features(self, schema_type, request):
schemas = request.getfixturevalue(schema_type)
if not isinstance(schemas, list):
schemas = [schemas]
for schema in schemas:
assert not has_xgrammar_unsupported_json_features(schema), (
f"Schema should be supported: {schema}"
)
class TestPR48416Regressions:
@pytest.mark.parametrize(
"schema",
[
{"type": ["number"], "multipleOf": 3},
{
"type": ["string", "null"],
"pattern": "^a+$",
"maxLength": 2,
},
{
"type": ["array", "null"],
"items": {"type": "integer"},
"uniqueItems": True,
},
{
"type": ["object", "null"],
"properties": {"Bad": {"type": "integer"}},
"propertyNames": {"pattern": "^[a-z]+$"},
},
],
)
def test_list_type_does_not_bypass_unsupported_feature_check(self, schema):
assert has_xgrammar_unsupported_json_features(schema)
@pytest.mark.parametrize(
"schema",
[
{"type": ["string", "null"]},
{"type": ["integer", "string"]},
{"type": ["array", "null"], "items": {"type": "string"}},
{
"type": ["object", "null"],
"propertyNames": {"pattern": "^[a-z_]+$"},
},
{
"type": ["object", "null"],
"patternProperties": {"^S": {"type": "string"}},
},
],
)
def test_supported_list_type_json_features(self, schema):
assert not has_xgrammar_unsupported_json_features(schema)
class TestIsGrammarAcceptString:
class TestPR42904Support:
def test_property_names_constrains_keys(self, property_names_schema):
assert grammar_accepts(property_names_schema, '{"score": 5}')
assert grammar_accepts(property_names_schema, '{"score": "seven"}')
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
def test_property_names_keeps_additional_properties_value_schema(
self, property_names_with_additional_properties_schema
):
schema = property_names_with_additional_properties_schema
assert grammar_accepts(schema, '{"score": 5}')
assert not grammar_accepts(schema, '{"Score": 5}')
assert not grammar_accepts(schema, '{"score": "seven"}')
def test_pattern_properties_constrains_keys_and_values(
self, pattern_properties_schema
):
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
class TestPR48115Regressions:
@pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F])
def test_choice_as_grammar_preserves_control_characters(self, codepoint):
choice = f"a{chr(codepoint)}f"
grammar = Grammar.from_ebnf(choice_as_grammar([choice]))
assert _is_grammar_accept_string(grammar, choice)
assert not _is_grammar_accept_string(grammar, "a")
assert not _is_grammar_accept_string(grammar, "af")
assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f")
@pytest.mark.parametrize(
("choice", "other"),
[
(r"a\nf", "a\nf"),
('a quote " and a backslash \\', 'a quote " and a backslash '),
("café", "cafe"),
("日本語", "日本"),
("😀", "😁"),
],
)
def test_choice_as_grammar_preserves_literal_choices(self, choice, other):
grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice]))
assert _is_grammar_accept_string(grammar, "yes")
assert _is_grammar_accept_string(grammar, choice)
assert not _is_grammar_accept_string(grammar, other)
assert not _is_grammar_accept_string(grammar, choice + "extra")
def test_validate_xgrammar_preserves_multiline_choice(self):
structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"])
validate_xgrammar_grammar(
SamplingParams(structured_outputs=structured_outputs)
)
assert structured_outputs.choice is None
grammar = Grammar.from_ebnf(structured_outputs.grammar)
assert _is_grammar_accept_string(grammar, "yes")
assert _is_grammar_accept_string(grammar, "no\nplease")
assert not _is_grammar_accept_string(grammar, r"no\nplease")
# ================================================
# Test XgrammarBackend functionality
# ================================================
class TestXgrammarBackend:
class TestPR58067Regressions:
"""Here we check behavior logic of our flag
disable_any_whitespace when used with xgrammar backend.
"""
_TOKENIZER = "openai-community/gpt2"
_VOCAB_SIZE = 50257
tokenizer: ClassVar[PreTrainedTokenizerBase]
@classmethod
@pytest.fixture(scope="class", autouse=True)
def _shared_tokenizer(cls):
cls.tokenizer = AutoTokenizer.from_pretrained(cls._TOKENIZER)
@classmethod
def _backend(cls, *, disable_any_whitespace: bool) -> XgrammarBackend:
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(
backend="xgrammar",
disable_any_whitespace=disable_any_whitespace,
)
)
return XgrammarBackend(
vllm_config, tokenizer=cls.tokenizer, vocab_size=cls._VOCAB_SIZE
)
@classmethod
def _accepts_json(cls, backend: XgrammarBackend, json_text: str) -> bool:
grammar = backend.compile_grammar(StructuredOutputOptions.JSON_OBJECT, "")
return grammar.accept_tokens("req", cls.tokenizer.encode(json_text))
def test_disable_any_whitespace_is_false(self):
"""Verify expected behavior of disable_any_whitespace=False"""
backend = self._backend(disable_any_whitespace=False)
# Check that grammar accepts json with NO space after colon
assert self._accepts_json(backend, '{"no_space_after_me":"yes_ofcourse"}')
# Check that grammar accepts json with space after colon
assert self._accepts_json(backend, '{"yes_space_after_me": true}')
# Check that grammar accepts json with NO space after comma
assert self._accepts_json(
backend,
'{"never_space_after_comma":"accepted","good_boy":":3"}',
)
# Check that grammar accepts json with space after comma
assert self._accepts_json(
backend,
'{"where_space_after_comma":42, "wow_thats_cool":4242}',
)
def test_disable_any_whitespace_is_true(self):
"""Verify expected behavior of disable_any_whitespace=True"""
backend = self._backend(disable_any_whitespace=True)
# Check that grammar accepts json with NO space after colon
assert self._accepts_json(backend, '{"no_space_after_me":"yes_ofcourse"}')
# Check that grammar rejects json with space after colon
assert not self._accepts_json(backend, '{"yes_space_after_me": true}')
# FIXME(arpera):
# Comma + separators=(",", ":") is wrong on xgrammar <=0.2.8
# Seehttps://github.com/mlc-ai/xgrammar/issues/945
# Re-enable both tests once vLLM pins xgrammar with the fix (>=0.2.9).
#
# # Check that grammar accepts json with NO space after comma
# assert self._accepts_json(
# backend,
# '{"never_space_after_comma":"accepted","good_boy":":3"}',
# )
#
# # Check that grammar rejects json with space after comma
# assert not self._accepts_json(
# backend,
# '{"where_space_after_comma":42, "wow_thats_cool":4242}',
# )

Some files were not shown because too many files have changed in this diff Show More