[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
class Matches(NamedTuple):
|
||||
# simple pointwise
|
||||
aiter_rms_quant_fusion: int = 0
|
||||
rms_quant_fusion: int = 0
|
||||
act_quant_fusion: int = 0
|
||||
norm_rope_fusion: int = 0
|
||||
attn_quant_fusion: int = 0
|
||||
# distributed
|
||||
ar_rms_fusion: int = 0
|
||||
aiter_ar_rms_fusion: int = 0
|
||||
|
||||
|
||||
class ModelFusionInfo(NamedTuple):
|
||||
model_name: str
|
||||
matches: Callable[[int], Matches]
|
||||
"""Given number of hidden layers, produces the matches object"""
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
|
||||
|
||||
|
||||
class AttentionBackendCase(NamedTuple):
|
||||
backend: AttentionBackendEnum
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
"""Additional args required for attn+quant fusion"""
|
||||
|
||||
|
||||
is_blackwell = lambda: current_platform.is_device_capability_family(100)
|
||||
"""Are we running on Blackwell, a lot of tests depend on it"""
|
||||
|
||||
|
||||
def nvfp4_kernel_exposes_input_quant_key() -> bool:
|
||||
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
|
||||
|
||||
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
|
||||
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
|
||||
output format and the cutedsl backend's expected input format.
|
||||
|
||||
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
|
||||
manual fusion.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fallback: assume no manual fusion support
|
||||
return False
|
||||
|
||||
|
||||
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
|
||||
"""Generate all combinations of custom ops for parametrization."""
|
||||
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
|
||||
for op_list in itertools.product(*custom_ops_lists):
|
||||
yield ",".join(op_list)
|
||||
|
||||
|
||||
# Quick inline validation
|
||||
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
|
||||
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
|
||||
"-quant_fp8,-rms_norm",
|
||||
"-quant_fp8,+rms_norm",
|
||||
"+quant_fp8,-rms_norm",
|
||||
"+quant_fp8,+rms_norm",
|
||||
]
|
||||
|
||||
|
||||
def has_cuda_graph_wrapper_metadata() -> bool:
|
||||
from importlib import import_module
|
||||
|
||||
try:
|
||||
module = import_module("torch._inductor.utils")
|
||||
module.CUDAGraphWrapperMetadata # noqa B018
|
||||
except AttributeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
INDUCTOR_GRAPH_PARTITION = [
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not has_cuda_graph_wrapper_metadata(),
|
||||
reason="torch version does not support Inductor partition",
|
||||
),
|
||||
id="inductor_partition",
|
||||
),
|
||||
pytest.param(False, id="dynamo_partition"),
|
||||
]
|
||||
|
||||
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_rms_quant_fusion": re.compile(
|
||||
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
# The combined pass can consume these sites before the standalone pass.
|
||||
"norm_rope_fusion": re.compile(
|
||||
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
|
||||
r"qk_norm_rope_kvcache_fusion.py:\d+] "
|
||||
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
|
||||
),
|
||||
"attn_quant_fusion": re.compile(
|
||||
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
|
||||
),
|
||||
"ar_rms_fusion": re.compile(
|
||||
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
|
||||
),
|
||||
"aiter_ar_rms_fusion": re.compile(
|
||||
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
|
||||
from .common import FUSION_LOG_PATTERNS, AttentionBackendCase, Matches
|
||||
|
||||
|
||||
def run_model(
|
||||
vllm_runner,
|
||||
compile_config: CompilationMode | CompilationConfig,
|
||||
model: str,
|
||||
**model_kwargs,
|
||||
):
|
||||
"""Run a model with the given compilation config for E2E fusion tests."""
|
||||
compilation_config = (
|
||||
compile_config
|
||||
if isinstance(compile_config, CompilationConfig)
|
||||
else CompilationConfig(mode=compile_config)
|
||||
)
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
# Allow override from model_kwargs
|
||||
model_kwargs = {"tensor_parallel_size": 1, **model_kwargs}
|
||||
model_kwargs = {"disable_custom_all_reduce": True, **model_kwargs}
|
||||
|
||||
# No cudagraphs by default
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# trust_remote_code/enable_chunked_prefill/block_size/kernel_config are
|
||||
# pinned to LLM()'s own defaults since VllmRunner overrides them for
|
||||
# other test needs (e.g. it forces block_size=16, which some attention
|
||||
# backends like FLASHINFER_MLA reject, and disables JIT warmup).
|
||||
model_kwargs.setdefault("kernel_config", {"enable_jit_warmup": True})
|
||||
with vllm_runner(
|
||||
model,
|
||||
compilation_config=compilation_config,
|
||||
trust_remote_code=False,
|
||||
enable_chunked_prefill=None,
|
||||
block_size=None,
|
||||
**model_kwargs,
|
||||
) as vllm_model:
|
||||
llm = vllm_model.llm
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
# Print the outputs.
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
|
||||
# Get the compile ranges endpoints after vllm config post init
|
||||
# in order to compute compile ranges correctly
|
||||
compilation_config.compile_ranges_endpoints = (
|
||||
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
|
||||
)
|
||||
|
||||
# Fetch match table from each worker via RPC and sum across workers.
|
||||
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
|
||||
"get_compilation_match_table"
|
||||
)
|
||||
combined: defaultdict[str, int] = defaultdict(int)
|
||||
for table in worker_tables:
|
||||
for k, v in table.items():
|
||||
combined[k] += v
|
||||
return dict(combined)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
def run(
|
||||
model_name: str,
|
||||
matches: Matches,
|
||||
model_kwargs: dict,
|
||||
attn_backend: AttentionBackendCase,
|
||||
compilation_config: dict,
|
||||
matches_check: list[str],
|
||||
use_deepgemm: bool = False,
|
||||
use_aiter: bool = False,
|
||||
tp_size: int = 1,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1" if use_aiter else "0")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
# Filter here to reduce code duplication
|
||||
backend_name = attn_backend.backend.name.lower()
|
||||
requires_mla = "deepseek" in model_name.lower()
|
||||
is_mla = "mla" in backend_name
|
||||
|
||||
if requires_mla != is_mla:
|
||||
pytest.skip(
|
||||
f"Incompatible model '{model_name}' and "
|
||||
f"attention backend '{attn_backend.backend.name}'"
|
||||
)
|
||||
|
||||
if backend_name == "rocm_attn" and model_name == "openai/gpt-oss-20b":
|
||||
pytest.skip(
|
||||
"ROCM_ATTN does not support attention sinks (required by gpt-oss-20b)"
|
||||
)
|
||||
|
||||
if attn_backend.backend.name == "FLASHINFER":
|
||||
from vllm.utils.flashinfer import supports_trtllm_attention
|
||||
|
||||
if not supports_trtllm_attention():
|
||||
matches = matches._replace(attn_quant_fusion=0)
|
||||
|
||||
# Disable, compile cache to make sure custom passes run.
|
||||
# Otherwise, we can't verify fusion happened through the logs.
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
# To capture subprocess logs, we need to know whether spawn or fork is used.
|
||||
# Force spawn as it is more general.
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
model_kwargs = {**attn_backend.model_kwargs, **model_kwargs}
|
||||
model_kwargs["attention_config"] = {"backend": attn_backend.backend.name}
|
||||
model_kwargs["tensor_parallel_size"] = tp_size
|
||||
|
||||
# Cap warmup memory: tests use small max_model_len (1024) but the
|
||||
# engine default max_num_batched_tokens is 16384. Warming up large
|
||||
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
|
||||
model_kwargs.setdefault("max_num_batched_tokens", 8192)
|
||||
|
||||
# Always compile the full graph instead of piecewise
|
||||
if not compilation_config["use_inductor_graph_partition"]:
|
||||
compilation_config["splitting_ops"] = []
|
||||
|
||||
full_compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
inductor_compile_config={"force_disable_caches": True},
|
||||
**compilation_config,
|
||||
)
|
||||
|
||||
with caplog_mp_spawn(logging.DEBUG) as log_holder:
|
||||
match_table = run_model(
|
||||
vllm_runner, full_compilation_config, model_name, **model_kwargs
|
||||
)
|
||||
|
||||
num_compile_ranges = len(full_compilation_config.get_compile_ranges())
|
||||
assert num_compile_ranges in [1, 2, 3]
|
||||
|
||||
print(f"Compile ranges: {full_compilation_config.get_compile_ranges()}")
|
||||
print("Fusion results:")
|
||||
|
||||
# Iterate through all so printing happens before asserting
|
||||
log_matches_dict = {}
|
||||
for match_name, pattern in FUSION_LOG_PATTERNS.items():
|
||||
log_matches_dict[match_name] = list(pattern.findall(log_holder.text))
|
||||
print(f"- {match_name}={','.join(log_matches_dict[match_name])}")
|
||||
|
||||
# Now check the matches
|
||||
for match_name in matches_check:
|
||||
log_matches = list(int(ms) for ms in log_matches_dict[match_name])
|
||||
|
||||
# AR+RMS skips the largest range.
|
||||
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
|
||||
num_ranges_activated = num_compile_ranges - 1
|
||||
else:
|
||||
num_ranges_activated = num_compile_ranges
|
||||
|
||||
# TODO: Remove log counting in unit tests
|
||||
# once all matchers implement VllmFusionPatternMatcherPass
|
||||
n_expected = tp_size * num_ranges_activated
|
||||
if match_name not in (
|
||||
"attn_quant_fusion",
|
||||
"act_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
):
|
||||
assert len(log_matches) == n_expected, (
|
||||
f"Could not find {n_expected} {match_name} "
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
expected_matches = getattr(matches, match_name)
|
||||
|
||||
if match_name == "norm_rope_fusion":
|
||||
# Opaque LayerName lets the combined pass consume short-range
|
||||
# sites before the standalone pass; count either owner.
|
||||
assert len(log_matches) >= n_expected
|
||||
assert all(m in (0, expected_matches) for m in log_matches)
|
||||
assert sum(log_matches) == expected_matches * n_expected, (
|
||||
f"{match_name} expected {expected_matches * n_expected} "
|
||||
f"sites across both passes, found: {log_matches}"
|
||||
)
|
||||
elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
|
||||
# AR+rms+quant takes precedence over rms+quant if activated.
|
||||
# That means we get full matching where ar+rms+quant was not
|
||||
# activated, and less where it was (only the smallest range).
|
||||
assert sum(m == expected_matches for m in log_matches) == tp_size * (
|
||||
num_ranges_activated - 1
|
||||
), "Expecting full rms+quant fusion where ar+rms+quant not activated"
|
||||
|
||||
assert all(
|
||||
expected_matches - matches.ar_rms_fusion <= m <= expected_matches
|
||||
for m in log_matches
|
||||
), (
|
||||
f"Expecting at least {expected_matches - matches.ar_rms_fusion} "
|
||||
f"where ar+rms+quant was activated"
|
||||
)
|
||||
elif match_name == "act_quant_fusion":
|
||||
actual_match = match_table.get("activation_quant_fusion_pass", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
f"Could not find {expected_matches * n_expected} "
|
||||
f"{match_name} (found {actual_match})."
|
||||
)
|
||||
elif match_name == "attn_quant_fusion":
|
||||
actual_match = match_table.get(
|
||||
"attn_quant_fusion", 0
|
||||
) + match_table.get("mla_attn_quant_fusion", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
f"Could not find {expected_matches * n_expected} "
|
||||
f"{match_name} (found {actual_match})."
|
||||
)
|
||||
else:
|
||||
expected_matches_list = [expected_matches] * n_expected
|
||||
assert sorted(log_matches) == expected_matches_list, (
|
||||
f"{match_name} expected: {expected_matches_list}, "
|
||||
f"found: {sorted(log_matches)}"
|
||||
)
|
||||
|
||||
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
|
||||
log_matches = re.findall(
|
||||
r"pass_manager.py:\d+] Skipping "
|
||||
r".*AllReduceFusionPass.* with compile range",
|
||||
log_holder.text,
|
||||
)
|
||||
|
||||
n_expected = tp_size * (num_compile_ranges - num_ranges_activated)
|
||||
assert len(log_matches) == n_expected, (
|
||||
f'Could not find {n_expected} "Skipping AllReduceFusionPass" '
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
return run
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
|
||||
def test_mp_reducer():
|
||||
"""Test that _reduce_config reducer is registered when AsyncLLM is instantiated
|
||||
without transformers_modules. This is a regression test for
|
||||
https://github.com/vllm-project/vllm/pull/18640.
|
||||
"""
|
||||
# Ensure transformers_modules is not in sys.modules
|
||||
if "transformers_modules" in sys.modules:
|
||||
del sys.modules["transformers_modules"]
|
||||
|
||||
with patch("multiprocessing.reducer.register") as mock_register:
|
||||
engine_args = AsyncEngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
max_model_len=32,
|
||||
gpu_memory_utilization=0.1,
|
||||
disable_log_stats=True,
|
||||
)
|
||||
|
||||
async_llm = AsyncLLM.from_engine_args(
|
||||
engine_args,
|
||||
start_engine_loop=False,
|
||||
)
|
||||
|
||||
assert mock_register.called, (
|
||||
"multiprocessing.reducer.register should have been called"
|
||||
)
|
||||
|
||||
vllm_config_registered = False
|
||||
for call_args in mock_register.call_args_list:
|
||||
# Verify that a reducer for VllmConfig was registered
|
||||
if len(call_args[0]) >= 2 and call_args[0][0] == VllmConfig:
|
||||
vllm_config_registered = True
|
||||
|
||||
reducer_func = call_args[0][1]
|
||||
assert callable(reducer_func), "Reducer function should be callable"
|
||||
break
|
||||
|
||||
assert vllm_config_registered, (
|
||||
"VllmConfig should have been registered to multiprocessing.reducer"
|
||||
)
|
||||
|
||||
async_llm.shutdown()
|
||||
@@ -1,31 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.llm import LLM
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
|
||||
def test_computed_prefix_blocks(model: str):
|
||||
# This test checks if the engine generates completions both with and
|
||||
# without optional detokenization, that detokenization includes text
|
||||
# and no-detokenization doesn't, and that both completions have the same
|
||||
# token_ids.
|
||||
prompt = (
|
||||
"You are a helpful assistant. How do I build a car from cardboard and "
|
||||
"paper clips? Is there an easy to follow video tutorial available "
|
||||
"online for free?"
|
||||
)
|
||||
|
||||
llm = LLM(model=model)
|
||||
sampling_params = SamplingParams(max_tokens=10, temperature=0.0, detokenize=False)
|
||||
|
||||
outputs_no_detokenization = llm.generate(prompt, sampling_params)[0].outputs[0]
|
||||
sampling_params.detokenize = True
|
||||
outputs_with_detokenization = llm.generate(prompt, sampling_params)[0].outputs[0]
|
||||
|
||||
assert outputs_no_detokenization.text == ""
|
||||
assert outputs_with_detokenization.text != ""
|
||||
assert outputs_no_detokenization.token_ids == outputs_with_detokenization.token_ids
|
||||
@@ -1,205 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Orchestration-level integration tests for RayExecutorV2."""
|
||||
|
||||
import gc
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
|
||||
|
||||
MODEL = "facebook/opt-125m"
|
||||
|
||||
|
||||
def _get_env_var(worker, name):
|
||||
return os.environ.get(name)
|
||||
|
||||
|
||||
def _ray_init():
|
||||
"""Start Ray with the project root on workers' PYTHONPATH.
|
||||
|
||||
Without this, workers cannot unpickle actor classes defined in the
|
||||
``tests`` package, causing FunctionActorManager to fall back to
|
||||
TemporaryActor which drops async method signatures."""
|
||||
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
||||
ray.init(
|
||||
ignore_reinit_error=True,
|
||||
runtime_env={"env_vars": {"PYTHONPATH": project_root}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ray_init():
|
||||
_ray_init()
|
||||
|
||||
|
||||
class _AsyncLLMActor:
|
||||
def start(self, pg, bundle_indices=None, ray_runtime_env=None):
|
||||
# Needed so collective_rpc can pickle _get_env_var over the
|
||||
# AsyncLLM -> EngineCore ZMQ boundary.
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
if bundle_indices is not None:
|
||||
os.environ["VLLM_RAY_BUNDLE_INDICES"] = bundle_indices
|
||||
else:
|
||||
os.environ.pop("VLLM_RAY_BUNDLE_INDICES", None)
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=2,
|
||||
distributed_executor_backend="ray",
|
||||
enforce_eager=True,
|
||||
max_model_len=256,
|
||||
gpu_memory_utilization=0.8,
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
vllm_config.parallel_config.placement_group = pg
|
||||
if ray_runtime_env is not None:
|
||||
vllm_config.parallel_config.ray_runtime_env = ray_runtime_env
|
||||
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
self.engine = AsyncLLM(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
log_requests=False,
|
||||
)
|
||||
|
||||
async def generate(self, prompt):
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
params = SamplingParams(max_tokens=16)
|
||||
result = None
|
||||
async for output in self.engine.generate(
|
||||
prompt, params, request_id="test_request_id"
|
||||
):
|
||||
result = output
|
||||
assert result is not None
|
||||
return result.outputs[0].text
|
||||
|
||||
async def generate_and_get_worker_envs(self, prompt, env_names):
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
params = SamplingParams(max_tokens=16)
|
||||
result = None
|
||||
async for output in self.engine.generate(
|
||||
prompt, params, request_id="test_request_id"
|
||||
):
|
||||
result = output
|
||||
assert result is not None
|
||||
text = result.outputs[0].text
|
||||
|
||||
env_results = {}
|
||||
for name in env_names:
|
||||
vals = await self.engine.collective_rpc(
|
||||
_get_env_var, timeout=10, args=(name,)
|
||||
)
|
||||
env_results[name] = vals
|
||||
return text, env_results
|
||||
|
||||
def shutdown(self):
|
||||
if engine := getattr(self, "engine", None):
|
||||
engine.shutdown()
|
||||
del self.engine
|
||||
gc.collect()
|
||||
|
||||
|
||||
AsyncLLMActor = ray.remote(num_cpus=0, max_concurrency=1)(_AsyncLLMActor)
|
||||
|
||||
|
||||
def test_multi_replicas(ray_init):
|
||||
pg1 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
|
||||
pg2 = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
|
||||
ray.get([pg1.ready(), pg2.ready()])
|
||||
|
||||
actor1 = AsyncLLMActor.remote()
|
||||
actor2 = AsyncLLMActor.remote()
|
||||
|
||||
ray.get(actor1.start.remote(pg1))
|
||||
ray.get(actor2.start.remote(pg2))
|
||||
|
||||
out1, out2 = ray.get(
|
||||
[
|
||||
actor1.generate.remote("Hello world"),
|
||||
actor2.generate.remote("Hello world"),
|
||||
]
|
||||
)
|
||||
assert len(out1) > 0
|
||||
assert len(out2) > 0
|
||||
|
||||
|
||||
def test_multi_replicas_with_bundle_indices(ray_init):
|
||||
pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 4, strategy="PACK")
|
||||
ray.get(pg.ready())
|
||||
|
||||
actor1 = AsyncLLMActor.remote()
|
||||
actor2 = AsyncLLMActor.remote()
|
||||
|
||||
ray.get(actor1.start.remote(pg, bundle_indices="2,1"))
|
||||
ray.get(actor2.start.remote(pg, bundle_indices="0,3"))
|
||||
|
||||
out1, out2 = ray.get(
|
||||
[
|
||||
actor1.generate.remote("Hello world"),
|
||||
actor2.generate.remote("Hello world"),
|
||||
]
|
||||
)
|
||||
assert len(out1) > 0
|
||||
assert len(out2) > 0
|
||||
|
||||
|
||||
def test_env_var_and_runtime_env_propagation():
|
||||
"""Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
|
||||
propagate to RayWorkerProc actors.
|
||||
"""
|
||||
sentinel_vars = {
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"HF_TOKEN": "test_sentinel_token",
|
||||
}
|
||||
for k, v in sentinel_vars.items():
|
||||
os.environ[k] = v
|
||||
|
||||
try:
|
||||
# Called directly (not via the ray_init fixture) because sentinel
|
||||
# env vars must be in os.environ before ray.init() so that Ray
|
||||
# worker processes inherit them.
|
||||
_ray_init()
|
||||
|
||||
pg = ray.util.placement_group([{"GPU": 1, "CPU": 1}] * 2, strategy="PACK")
|
||||
ray.get(pg.ready())
|
||||
|
||||
# Include the project root so that RayWorkerProc actors can
|
||||
# unpickle _get_env_var.
|
||||
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
||||
ray_runtime_env = {
|
||||
"env_vars": {
|
||||
"RAY_RUNTIME_ENV_TEST": "ray_runtime_env",
|
||||
"PYTHONPATH": project_root,
|
||||
},
|
||||
}
|
||||
|
||||
actor = AsyncLLMActor.remote()
|
||||
ray.get(actor.start.remote(pg, ray_runtime_env=ray_runtime_env))
|
||||
|
||||
all_env_names = list(sentinel_vars) + ["RAY_RUNTIME_ENV_TEST"]
|
||||
text, env_results = ray.get(
|
||||
actor.generate_and_get_worker_envs.remote("Hello world", all_env_names)
|
||||
)
|
||||
assert len(text) > 0
|
||||
|
||||
for name, expected in sentinel_vars.items():
|
||||
for val in env_results[name]:
|
||||
assert val == expected
|
||||
|
||||
for val in env_results["RAY_RUNTIME_ENV_TEST"]:
|
||||
assert val == "ray_runtime_env"
|
||||
|
||||
finally:
|
||||
for k in sentinel_vars:
|
||||
os.environ.pop(k, None)
|
||||
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from tests.v1.utils import check_request_balancing
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
DP_SIZE = os.getenv("DP_SIZE", "1")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--api-server-count",
|
||||
"4",
|
||||
"--data_parallel_size",
|
||||
DP_SIZE,
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_completion(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
async def make_request():
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
|
||||
choice = completion.choices[0]
|
||||
# The exact number of tokens can vary slightly with temperature=1.0,
|
||||
# so we check for a reasonable minimum length.
|
||||
assert len(choice.text) >= 1
|
||||
# Finish reason might not always be 'length' if the model finishes early
|
||||
# or due to other reasons, especially with high temperature.
|
||||
# So, we'll accept 'length' or 'stop'.
|
||||
assert choice.finish_reason in ("length", "stop")
|
||||
|
||||
# Token counts can also vary, so we check they are positive.
|
||||
assert completion.usage.completion_tokens > 0
|
||||
assert completion.usage.prompt_tokens > 0
|
||||
assert completion.usage.total_tokens > 0
|
||||
return completion
|
||||
|
||||
# Test single request
|
||||
result = await make_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_streaming(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request():
|
||||
# Perform a non-streaming request to get the expected full output
|
||||
single_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
# Perform the streaming request
|
||||
stream = await client.completions.create(
|
||||
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
last_chunk = None
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
last_chunk = chunk # Keep track of the last chunk
|
||||
|
||||
# finish reason should only return in the last block for OpenAI API
|
||||
assert finish_reason_count == 1, "Finish reason should appear exactly once."
|
||||
assert last_chunk is not None, "Stream should have yielded at least one chunk."
|
||||
assert last_chunk.choices[0].finish_reason == "length", (
|
||||
"Finish reason should be 'length'."
|
||||
)
|
||||
# Check that the combined text matches the non-streamed version.
|
||||
assert "".join(chunks) == single_output, (
|
||||
"Streamed output should match non-streamed output."
|
||||
)
|
||||
return True # Indicate success for this request
|
||||
|
||||
# Test single request
|
||||
result = await make_streaming_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def vision_llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
"microsoft/Phi-3.5-vision-instruct",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
seed=0,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
|
||||
)
|
||||
def test_chat_multi_image(vision_llm, image_urls: list[str]):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*(
|
||||
{"type": "image_url", "image_url": {"url": image_url}}
|
||||
for image_url in image_urls
|
||||
),
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
outputs = vision_llm.chat(messages)
|
||||
assert len(outputs) >= 0
|
||||
@@ -1,643 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
MODEL_NAME_SHORT = "gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
|
||||
BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
task = "generate"
|
||||
runner_type = "generate"
|
||||
model = MODEL_NAME
|
||||
tokenizer = MODEL_NAME
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
revision = None
|
||||
code_revision = None
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = MockHFConfig()
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
def _build_renderer(model_config: MockModelConfig):
|
||||
return HfRenderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_chat.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_chat
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_non_stream():
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
with pytest.raises(GenerationError):
|
||||
await serving_chat.create_chat_completion(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chat_keeps_mm_cache_for_engine_execution():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
result = await serving_chat.render_chat_request(request)
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert (
|
||||
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream(include_usage: bool):
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output_1 = CompletionOutput(
|
||||
index=0,
|
||||
text="Hello",
|
||||
token_ids=[100],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=None,
|
||||
)
|
||||
|
||||
request_output_1 = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output_1],
|
||||
finished=False,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
completion_output_2 = CompletionOutput(
|
||||
index=0,
|
||||
text="Hello",
|
||||
token_ids=[100],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output_2 = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output_2],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output_1
|
||||
yield request_output_2
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream_first_chunk(include_usage: bool):
|
||||
"""An error before the first token must reach the streaming client."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_content",
|
||||
[
|
||||
[{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}],
|
||||
[{"image_url": {"url": "https://example.com/image.jpg"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_image(image_content):
|
||||
"""Test that system messages with image content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": image_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "image_url" in call_args
|
||||
|
||||
|
||||
def test_system_message_accepts_text():
|
||||
"""Test that system messages can contain text content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "system"
|
||||
|
||||
|
||||
def test_system_message_accepts_text_array():
|
||||
"""Test that system messages can contain an array with text content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "You are a helpful assistant."}],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "system"
|
||||
|
||||
|
||||
def test_user_message_accepts_image():
|
||||
"""Test that user messages can still contain image content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.jpg"},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "user"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"audio_content",
|
||||
[
|
||||
[
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "base64data", "format": "wav"},
|
||||
}
|
||||
],
|
||||
[{"input_audio": {"data": "base64data", "format": "wav"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_audio(audio_content):
|
||||
"""Test that system messages with audio content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": audio_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "input_audio" in call_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"video_content",
|
||||
[
|
||||
[{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}],
|
||||
[{"video_url": {"url": "https://example.com/video.mp4"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_video(video_content):
|
||||
"""Test that system messages with video content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": video_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "video_url" in call_args
|
||||
|
||||
|
||||
def test_json_schema_response_format_missing_schema():
|
||||
"""When response_format type is 'json_schema' but the json_schema field
|
||||
is not provided, request construction should raise a validation error
|
||||
so the API returns 400 instead of 500."""
|
||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "json_schema"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
mm_processor_kwargs={
|
||||
"patch_size": 1,
|
||||
"vision_min_num_patches": 3_000_000_000,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request mm_processor_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_media_io_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
media_io_kwargs={"video": {"num_frames": 1_000_000}},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request media_io_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs={"use_audio_in_video": True},
|
||||
media_io_kwargs={"video": {"num_frames": 4}},
|
||||
trust_request_mm_kwargs=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_structural_tag_response_format_invalid(format_value):
|
||||
"""Malformed structural tags should be rejected during request validation."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_batch_structural_tag_response_format_invalid(format_value):
|
||||
"""Batch chat should reject malformed structural tags at request parsing."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
BatchChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[[{"role": "user", "content": "hello"}]],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"structural_tag",
|
||||
[
|
||||
"not json",
|
||||
"",
|
||||
# json.loads raises a plain ValueError, not JSONDecodeError
|
||||
'{"type": "structural_tag", "x": ' + "1" * 5000 + "}",
|
||||
],
|
||||
)
|
||||
def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
"""Malformed direct structured_outputs structural tags should be rejected."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid structured_outputs structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
structured_outputs={"structural_tag": structural_tag},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["response_format", "structured_outputs"])
|
||||
def test_deeply_nested_structural_tag_rejected(field):
|
||||
"""Structural tags are compiled during request parsing, so a deeply nested
|
||||
one must be rejected there instead of blocking the server while converting,
|
||||
and reported as too deep rather than malformed."""
|
||||
schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200
|
||||
tag = (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
value = json.loads(tag) if field == "response_format" else {"structural_tag": tag}
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
**{field: value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
||||
def test_non_numeric_logprobs_rejected(field_name):
|
||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
**{field_name: "2"},
|
||||
)
|
||||
@@ -1,304 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
|
||||
# downloading lora to test lora requests
|
||||
from openai import BadRequestError
|
||||
from transformers import AutoConfig
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
LORA_SERVING_MODEL_NAME = "opt125m-lora"
|
||||
|
||||
CONFIG = AutoConfig.from_pretrained(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=["use-lora"])
|
||||
def default_server_args(
|
||||
request: pytest.FixtureRequest, opt125_lora_files: str
|
||||
) -> list[str]:
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
if request.param == "use-lora":
|
||||
lora_module_1 = {
|
||||
"name": LORA_SERVING_MODEL_NAME,
|
||||
"path": opt125_lora_files,
|
||||
"base_model_name": MODEL_NAME,
|
||||
}
|
||||
|
||||
args.extend(
|
||||
[
|
||||
"--enable-lora",
|
||||
"--lora-module",
|
||||
json.dumps(lora_module_1),
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
EXAMPLE_PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"What is an LLM?",
|
||||
]
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor):
|
||||
buffer = io.BytesIO()
|
||||
torch.save(embeds, buffer)
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def example_prompt_embeds(hf_runner):
|
||||
"""Create example embeddings and return them as base64 encoded string."""
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
example_embeddings = hf_model.get_prompt_embeddings(EXAMPLE_PROMPTS)
|
||||
|
||||
return [_encode_embeds(item) for item in example_embeddings]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_with_prompt_embeds(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client_with_prompt_embeds(server_with_prompt_embeds):
|
||||
async with server_with_prompt_embeds.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_with_prompt_embeds(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
):
|
||||
encoded_embeds, encoded_embeds2 = example_prompt_embeds
|
||||
|
||||
# Test case: Single prompt embeds input
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
assert len(completion.choices[0].text) >= 1
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
# Test case: batch completion with prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
assert len(completion.choices) == 2
|
||||
assert len(completion.choices[0].text) >= 1
|
||||
assert len(completion.choices[1].text) >= 1
|
||||
|
||||
# Test case: streaming with prompt_embeds
|
||||
single_completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
stream = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
chunks = []
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
assert finish_reason_count == 1
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
assert chunk.choices[0].text
|
||||
assert "".join(chunks) == single_output
|
||||
|
||||
# Test case: batch streaming with prompt_embeds
|
||||
stream = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
chunks_stream_embeds: list[list[str]] = [[], []]
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
chunks_stream_embeds[chunk.choices[0].index].append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
assert finish_reason_count == 2
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
assert chunk.choices[0].text
|
||||
assert len(chunks_stream_embeds[0]) > 0
|
||||
assert len(chunks_stream_embeds[1]) > 0
|
||||
|
||||
# Test case: mixed text and prompt_embeds
|
||||
completion_mixed = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt="This is a prompt",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
assert len(completion.choices) == 2
|
||||
completion_text_only = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt="This is a prompt",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
completion_embeds_only = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
# Embeddings responses should be handled first
|
||||
assert completion_mixed.choices[0].text == completion_embeds_only.choices[0].text
|
||||
assert completion_mixed.choices[1].text == completion_text_only.choices[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_errors_with_prompt_embeds(
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
# Test error case: invalid prompt_embeds
|
||||
with pytest.raises(BadRequestError):
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
prompt=None,
|
||||
model=model_name,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": "invalid_base64"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logprobs_arg", [1, 0])
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_with_logprobs_and_prompt_embeds(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
logprobs_arg: int,
|
||||
model_name: str,
|
||||
):
|
||||
encoded_embeds, encoded_embeds2 = example_prompt_embeds
|
||||
|
||||
# Test case: Logprobs using prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
echo=False,
|
||||
logprobs=logprobs_arg,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
|
||||
logprobs = completion.choices[0].logprobs
|
||||
assert logprobs is not None
|
||||
assert len(logprobs.text_offset) == 5
|
||||
assert len(logprobs.token_logprobs) == 5
|
||||
assert len(logprobs.top_logprobs) == 5
|
||||
for top_logprobs in logprobs.top_logprobs[1:]:
|
||||
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
|
||||
assert len(logprobs.tokens) == 5
|
||||
|
||||
# Test case: Log probs with batch completion and prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
echo=False,
|
||||
logprobs=logprobs_arg,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
|
||||
assert len(completion.choices) == 2
|
||||
for choice in completion.choices:
|
||||
logprobs = choice.logprobs
|
||||
assert logprobs is not None
|
||||
assert len(logprobs.text_offset) == 5
|
||||
assert len(logprobs.token_logprobs) == 5
|
||||
assert len(logprobs.top_logprobs) == 5
|
||||
for top_logprobs in logprobs.top_logprobs[1:]:
|
||||
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
|
||||
assert len(logprobs.tokens) == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_logprobs_raises_error(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
):
|
||||
encoded_embeds, _ = example_prompt_embeds
|
||||
|
||||
with pytest.raises(BadRequestError, match="not compatible"):
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds, "prompt_logprobs": True},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_prompt_embeds(
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
) -> None:
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": []},
|
||||
)
|
||||
@@ -1,256 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
|
||||
|
||||
MOCK_RESOLVER_NAME = "mock_test_resolver"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
"""Minimal mock ModelConfig for testing."""
|
||||
|
||||
model: str = MODEL_NAME
|
||||
runner_type = "generate"
|
||||
tokenizer: str = MODEL_NAME
|
||||
trust_remote_code: bool = False
|
||||
tokenizer_mode: str = "auto"
|
||||
max_model_len: int = 100
|
||||
tokenizer_revision: str | None = None
|
||||
multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)
|
||||
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
class MockLoRAResolver(LoRAResolver):
|
||||
async def resolve_lora(
|
||||
self, base_model_name: str, lora_name: str
|
||||
) -> LoRARequest | None:
|
||||
if lora_name == "test-lora":
|
||||
return LoRARequest(
|
||||
lora_name="test-lora",
|
||||
lora_int_id=1,
|
||||
lora_path="/fake/path/test-lora",
|
||||
)
|
||||
elif lora_name == "invalid-lora":
|
||||
return LoRARequest(
|
||||
lora_name="invalid-lora",
|
||||
lora_int_id=2,
|
||||
lora_path="/fake/path/invalid-lora",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def register_mock_resolver():
|
||||
"""Fixture to register and unregister the mock LoRA resolver."""
|
||||
resolver = MockLoRAResolver()
|
||||
LoRAResolverRegistry.register_resolver(MOCK_RESOLVER_NAME, resolver)
|
||||
yield
|
||||
# Cleanup: remove the resolver after the test runs
|
||||
if MOCK_RESOLVER_NAME in LoRAResolverRegistry.resolvers:
|
||||
del LoRAResolverRegistry.resolvers[MOCK_RESOLVER_NAME]
|
||||
|
||||
|
||||
def _build_renderer(model_config: MockModelConfig):
|
||||
return HfRenderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_serving_setup():
|
||||
"""Provides a mocked engine and serving completion instance."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
|
||||
async def mock_add_lora_side_effect(lora_request: LoRARequest):
|
||||
"""Simulate engine behavior when adding LoRAs."""
|
||||
if lora_request.lora_name == "test-lora":
|
||||
# Simulate successful addition
|
||||
return True
|
||||
if lora_request.lora_name == "invalid-lora":
|
||||
# Simulate failure during addition (e.g. invalid format)
|
||||
raise ValueError(f"Simulated failure adding LoRA: {lora_request.lora_name}")
|
||||
return True
|
||||
|
||||
mock_engine.add_lora = AsyncMock(side_effect=mock_add_lora_side_effect)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
for _ in []:
|
||||
yield _
|
||||
|
||||
mock_engine.generate = MagicMock(spec=AsyncLLM.generate, side_effect=mock_generate)
|
||||
|
||||
mock_engine.generate.reset_mock()
|
||||
mock_engine.add_lora.reset_mock()
|
||||
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
models = OpenAIServingModels(
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
serving_completion = OpenAIServingCompletion(
|
||||
mock_engine, models, online_renderer=online_renderer, request_logger=None
|
||||
)
|
||||
|
||||
return mock_engine, serving_completion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_with_lora_resolver(mock_serving_setup, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
lora_model_name = "test-lora"
|
||||
req_found = CompletionRequest(
|
||||
model=lora_model_name,
|
||||
prompt="Generate with LoRA",
|
||||
)
|
||||
|
||||
# Suppress potential errors during the mocked generate call,
|
||||
# as we are primarily checking for add_lora and generate calls
|
||||
with suppress(Exception):
|
||||
await serving_completion.create_completion(req_found)
|
||||
|
||||
mock_engine.add_lora.assert_awaited_once()
|
||||
called_lora_request = mock_engine.add_lora.call_args[0][0]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == lora_model_name
|
||||
|
||||
mock_engine.generate.assert_called_once()
|
||||
called_lora_request = mock_engine.generate.call_args[1]["lora_request"]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == lora_model_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_resolver_not_found(mock_serving_setup, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
non_existent_model = "non-existent-lora-adapter"
|
||||
req = CompletionRequest(
|
||||
model=non_existent_model,
|
||||
prompt="what is 1+1?",
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(req)
|
||||
|
||||
mock_engine.add_lora.assert_not_awaited()
|
||||
mock_engine.generate.assert_not_called()
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == HTTPStatus.NOT_FOUND.value
|
||||
assert non_existent_model in response.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_resolver_add_lora_fails(
|
||||
mock_serving_setup, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
invalid_model = "invalid-lora"
|
||||
req = CompletionRequest(
|
||||
model=invalid_model,
|
||||
prompt="what is 1+1?",
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(req)
|
||||
|
||||
# Assert add_lora was called before the failure
|
||||
mock_engine.add_lora.assert_awaited_once()
|
||||
called_lora_request = mock_engine.add_lora.call_args[0][0]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == invalid_model
|
||||
|
||||
# Assert generate was *not* called due to the failure
|
||||
mock_engine.generate.assert_not_called()
|
||||
|
||||
# Assert the correct error response
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST.value
|
||||
assert invalid_model in response.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_flag_not_set(mock_serving_setup):
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
lora_model_name = "test-lora"
|
||||
req_found = CompletionRequest(
|
||||
model=lora_model_name,
|
||||
prompt="Generate with LoRA",
|
||||
)
|
||||
|
||||
await serving_completion.create_completion(req_found)
|
||||
|
||||
mock_engine.add_lora.assert_not_called()
|
||||
mock_engine.generate.assert_not_called()
|
||||
@@ -396,7 +396,6 @@ def server_with_store(default_server_args):
|
||||
default_server_args,
|
||||
env_dict={
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
class TestWorkerExtension:
|
||||
def get_model_name(self) -> str:
|
||||
"""Test non-pydantic return type."""
|
||||
return MODEL_NAME
|
||||
|
||||
def echo_args_kwargs(self, *args, **kwargs) -> dict[str, Any]:
|
||||
"""Echo back both args and kwargs."""
|
||||
return dict(
|
||||
args=list(args),
|
||||
kwargs=kwargs,
|
||||
total_items=len(args) + len(kwargs),
|
||||
)
|
||||
|
||||
def return_none(self, *args, **kwargs) -> None:
|
||||
"""Test method that does not return anything."""
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--worker-extension-cls",
|
||||
"tests.entrypoints.serve.dev.rpc.test_collective_rpc.TestWorkerExtension",
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_get_model_name(server):
|
||||
"""Test basic response."""
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"), json={"method": "get_model_name"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
assert "results" in results
|
||||
assert results["results"] == [MODEL_NAME]
|
||||
|
||||
|
||||
def test_return_none(server):
|
||||
"""Test return none."""
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"), json={"method": "return_none"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
assert results["results"] == [None]
|
||||
|
||||
|
||||
def test_echo_args_kwargs(server):
|
||||
"""Test args, kwargs, and dict response."""
|
||||
args = ["arg1", "arg2"]
|
||||
kwargs = {"key1": "value1", "key2": "value2"}
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"),
|
||||
json={"method": "echo_args_kwargs", "args": args, "kwargs": kwargs},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
result = results["results"][0]
|
||||
assert result["args"] == args
|
||||
assert result["kwargs"] == kwargs
|
||||
assert result["total_items"] == len(args) + len(kwargs)
|
||||
@@ -70,19 +70,17 @@ class MockModelConfig:
|
||||
|
||||
|
||||
@pytest.fixture(params=get_args(SupportedTask))
|
||||
def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]:
|
||||
def task_routes(request) -> tuple[str, list[tuple[str, list[str]]]]:
|
||||
"""For each supported task, build an app with only that task's routers,
|
||||
extract all routes, and return the task name and routes."""
|
||||
task = request.param
|
||||
# Enable development mode to register all routes (including dev-only routes).
|
||||
monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1")
|
||||
|
||||
app = FastAPI()
|
||||
args = Namespace()
|
||||
app.state = Namespace()
|
||||
app.state.args = args
|
||||
|
||||
# Register routers for this specific task (development mode already enabled).
|
||||
# Register routers for this specific task.
|
||||
register_api_routers(
|
||||
args,
|
||||
app,
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from argparse import Namespace
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.tokenize.api_router import attach_router
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
task = "generate"
|
||||
runner_type = "generate"
|
||||
model = MODEL_NAME
|
||||
tokenizer = MODEL_NAME
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = MockHFConfig()
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
return ServingTokenization(
|
||||
models,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=(
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
)
|
||||
|
||||
request = TokenizeChatRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
request = TokenizeCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
class TestDetokenizeClientErrorResponses:
|
||||
"""Client-caused errors from /detokenize are 4xx, not 500.
|
||||
|
||||
/detokenize used to wrap ``create_detokenize`` in a blanket
|
||||
``except Exception`` that reported every failure as 500, while its
|
||||
sibling /tokenize let exceptions propagate to the global exception
|
||||
handlers, which map client-caused errors to 4xx and keep genuine
|
||||
server errors at 500. These tests pin the shared contract for both
|
||||
endpoints. See #52246 for the same fix in the Anthropic entrypoint.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _make_api_app(handler: MagicMock) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.state.serving_tokenization = handler
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
return app
|
||||
|
||||
def _post(self, handler: MagicMock, path: str = "/detokenize"):
|
||||
app = self._make_api_app(handler)
|
||||
body: dict[str, Any] = {"model": "test-model"}
|
||||
if path == "/detokenize":
|
||||
body["tokens"] = [1, 2, 3]
|
||||
else:
|
||||
body["prompt"] = "Hello"
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
return client.post(path, json=body)
|
||||
|
||||
def test_vllm_validation_error_returns_bad_request(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = VLLMValidationError(
|
||||
"invalid token ids", parameter="tokens"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "BadRequestError"
|
||||
assert error["param"] == "tokens"
|
||||
|
||||
def test_vllm_not_found_error_returns_not_found(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = VLLMNotFoundError(
|
||||
"LoRA adapter nonexistent-lora not found"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
assert response.json()["error"]["type"] == "NotFoundError"
|
||||
|
||||
def test_value_error_returns_bad_request(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = ValueError("bad input")
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
def test_overflow_error_returns_bad_request(self):
|
||||
"""Fast tokenizers raise OverflowError for out-of-range token ids.
|
||||
|
||||
This used to be special-cased into a RequestValidationError; the
|
||||
global OverflowError handler keeps the same 400 status code.
|
||||
"""
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = OverflowError(
|
||||
"out of range integral type conversion attempted"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
def test_server_error_still_returns_internal_server_error(self):
|
||||
"""Genuine server bugs keep the existing 500 behaviour."""
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = RuntimeError("boom")
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
assert response.json()["error"]["type"] == "InternalServerError"
|
||||
|
||||
def test_tokenize_and_detokenize_agree_on_client_errors(self):
|
||||
"""The two sibling endpoints report the same client error alike."""
|
||||
statuses = {}
|
||||
for path, method in (
|
||||
("/tokenize", "create_tokenize"),
|
||||
("/detokenize", "create_detokenize"),
|
||||
):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
getattr(handler, method).side_effect = VLLMValidationError(
|
||||
"invalid input", parameter="model"
|
||||
)
|
||||
statuses[path] = self._post(handler, path=path).status_code
|
||||
|
||||
assert statuses["/tokenize"] == HTTPStatus.BAD_REQUEST
|
||||
assert statuses["/detokenize"] == HTTPStatus.BAD_REQUEST
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils import jit_monitor
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_monitor():
|
||||
"""Reset global monitor state between tests.
|
||||
|
||||
``activate()`` installs process-global hooks and flips module globals, so
|
||||
without this every test would inherit the previous test's monitor state.
|
||||
"""
|
||||
|
||||
def reset():
|
||||
jit_monitor._active = False
|
||||
jit_monitor._mode = "warn"
|
||||
jit_monitor._verbose = False
|
||||
jit_monitor._cutedsl_hook_installed = False
|
||||
jit_monitor._tilelang_hook_installed = False
|
||||
jit_monitor._tilelang_jitimpl_compile_depth = 0
|
||||
|
||||
reset()
|
||||
yield
|
||||
reset()
|
||||
@@ -1,19 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
create_kv_caches_with_random,
|
||||
create_kv_caches_with_random_flash,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kv_cache_factory():
|
||||
return create_kv_caches_with_random
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kv_cache_factory_flashinfer():
|
||||
return create_kv_caches_with_random_flash
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
|
||||
from .common import Config
|
||||
from .mk_objects import (
|
||||
MK_ALL_PREPARE_FINALIZE_TYPES,
|
||||
MK_FUSED_EXPERT_TYPES,
|
||||
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES,
|
||||
)
|
||||
|
||||
|
||||
def make_config_arg_parser(description: str):
|
||||
def to_pf_class_type(s: str) -> mk.FusedMoEPrepareAndFinalizeModular:
|
||||
for pf in MK_ALL_PREPARE_FINALIZE_TYPES:
|
||||
if pf.__name__ == s:
|
||||
return pf
|
||||
raise ValueError(f"Cannot find a PrepareFinalize type that matches {s}")
|
||||
|
||||
def to_experts_class_type(s: str) -> mk.FusedMoEExpertsModular:
|
||||
for fe in MK_FUSED_EXPERT_TYPES:
|
||||
if fe.__name__ == s:
|
||||
return fe
|
||||
raise ValueError(f"Cannot find a FusedExperts type that matches {s}")
|
||||
|
||||
def to_quant_torch_dtype(s: str) -> torch.dtype:
|
||||
if s == "torch.float8_e4m3fn":
|
||||
return torch.float8_e4m3fn
|
||||
raise ValueError(f"Unsupported quant type {s}")
|
||||
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
|
||||
parser.add_argument(
|
||||
"--world-size",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of ranks that participate in all2all",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pf-type",
|
||||
type=to_pf_class_type,
|
||||
required=True,
|
||||
help=(
|
||||
"Choose a PrepareFinalize Type : "
|
||||
f"{[x.__name__ for x in MK_ALL_PREPARE_FINALIZE_TYPES]}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--experts-type",
|
||||
type=to_experts_class_type,
|
||||
required=True,
|
||||
help=(
|
||||
f"Choose a FusedExpert type : {[x.__name__ for x in MK_FUSED_EXPERT_TYPES]}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m",
|
||||
nargs="+",
|
||||
type=int,
|
||||
default=[64],
|
||||
help="num tokens per rank",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k",
|
||||
type=int,
|
||||
default=7168,
|
||||
help="hidden-size",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
type=int,
|
||||
default=1024,
|
||||
help="N dimension of the first fused-moe matmul",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-experts", type=int, default=32, help="Global num experts"
|
||||
)
|
||||
parser.add_argument("--topk", nargs="+", type=int, default=[4, 1], help="num topk")
|
||||
|
||||
# Quant args
|
||||
parser.add_argument(
|
||||
"--quant-dtype", type=to_quant_torch_dtype, help="Quant datatype"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per-token-quantized-activations",
|
||||
action="store_true",
|
||||
help=("The input activations must be per-token quantized"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per-channel-quantized-weights",
|
||||
action="store_true",
|
||||
help="The weights must be per-channel quantized.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--block-shape", nargs="+", type=int, help="Quantization block shape"
|
||||
)
|
||||
|
||||
# Torch trace profile generation args
|
||||
parser.add_argument(
|
||||
"--torch-trace-dir-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Get torch trace for single execution",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def _validate_args(args: argparse.Namespace):
|
||||
if args.quant_dtype is not None:
|
||||
assert args.quant_dtype == torch.float8_e4m3fn
|
||||
if args.block_shape is not None:
|
||||
assert len(args.block_shape) == 2, (
|
||||
f"block shape must have 2 elements. got {args.block_shape}"
|
||||
)
|
||||
|
||||
if args.experts_type in MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES:
|
||||
assert args.world_size == 1, "Single GPU objects need world size set to 1"
|
||||
|
||||
if args.torch_trace_dir_path is not None:
|
||||
from pathlib import Path
|
||||
|
||||
assert Path(args.torch_trace_dir_path).is_dir(), (
|
||||
f"Please create {args.torch_trace_dir_path}"
|
||||
)
|
||||
|
||||
|
||||
def make_config(args: argparse.Namespace) -> Config:
|
||||
_validate_args(args)
|
||||
|
||||
quant_config = None
|
||||
if args.quant_dtype is not None:
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype=args.quant_dtype,
|
||||
per_act_token_quant=args.per_token_quantized_activations,
|
||||
per_out_ch_quant=args.per_channel_quantized_weights,
|
||||
block_shape=args.block_shape,
|
||||
)
|
||||
|
||||
return Config(
|
||||
Ms=args.m,
|
||||
K=args.k,
|
||||
N=args.n,
|
||||
E=args.num_experts,
|
||||
topks=args.topk,
|
||||
dtype=torch.bfloat16, # hard-code
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=args.pf_type,
|
||||
fused_experts_type=args.experts_type,
|
||||
world_size=args.world_size,
|
||||
torch_trace_dir_path=args.torch_trace_dir_path,
|
||||
)
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from enum import Enum
|
||||
from itertools import product
|
||||
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.config import FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
from .common import (
|
||||
Config,
|
||||
RankTensors,
|
||||
WeightTensors,
|
||||
reference_moe_impl,
|
||||
run_modular_kernel,
|
||||
)
|
||||
from .mk_objects import (
|
||||
MK_FUSED_EXPERT_TYPES,
|
||||
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES,
|
||||
MK_QUANT_CONFIGS,
|
||||
)
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
|
||||
|
||||
|
||||
class Result(Enum):
|
||||
PASS = 1
|
||||
FAIL = 2
|
||||
SKIP = 3
|
||||
|
||||
|
||||
def rank_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
):
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
# get weights to this device
|
||||
weights.to_current_device()
|
||||
|
||||
Ms = config.Ms
|
||||
assert isinstance(Ms, list)
|
||||
TOPKs = config.topks
|
||||
assert isinstance(TOPKs, list)
|
||||
|
||||
for m, topk in product(Ms, TOPKs):
|
||||
print(f"Running m={m}, topk={topk} ...")
|
||||
# override m and topk
|
||||
cfgx = copy.deepcopy(config)
|
||||
cfgx.Ms = m
|
||||
cfgx.topks = topk
|
||||
|
||||
# inputs for rank
|
||||
rank_tensors = RankTensors.make(cfgx, pgi)
|
||||
|
||||
# modular kernel out
|
||||
mk_out = run_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = reference_moe_impl(cfgx, weights, rank_tensors)
|
||||
|
||||
torch.testing.assert_close(ref_out, mk_out, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def make_feature_matrix(csv_file_path: str):
|
||||
from dataclasses import asdict
|
||||
|
||||
import pandas as pd
|
||||
|
||||
def add_to_results(
|
||||
config: Config, success: Result, results_df: pd.DataFrame | None = None
|
||||
):
|
||||
config_dict = asdict(config)
|
||||
config_dict["prepare_finalize_type"] = config_dict[
|
||||
"prepare_finalize_type"
|
||||
].__name__
|
||||
config_dict["fused_experts_type"] = config_dict["fused_experts_type"].__name__
|
||||
config_dict["per_tensor_act_quant"] = config.is_per_tensor_act_quant
|
||||
quant_config_dict = config_dict["quant_config"]
|
||||
del config_dict["quant_config"]
|
||||
if quant_config_dict is None:
|
||||
quant_config = FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
quant_config_dict = asdict(quant_config)
|
||||
|
||||
config_dict |= quant_config_dict
|
||||
result_dict = config_dict | {"success": success.name}
|
||||
|
||||
result_df = pd.DataFrame([result_dict])
|
||||
if results_df is None:
|
||||
results_df = result_df
|
||||
else:
|
||||
results_df = pd.concat([results_df, result_df], ignore_index=True)
|
||||
|
||||
return results_df
|
||||
|
||||
Ms = [64]
|
||||
Ks = [7168] # hidden sizes
|
||||
Ns = [2048]
|
||||
TOPKs = [[4, 1]]
|
||||
Es = [32]
|
||||
DTYPEs = [torch.bfloat16]
|
||||
PF_TYPES = MK_MULTI_GPU_PREPARE_FINALIZE_TYPES
|
||||
FE_TYPES = MK_FUSED_EXPERT_TYPES
|
||||
Q_TYPES = MK_QUANT_CONFIGS
|
||||
|
||||
combinations = list(
|
||||
product(Ms, Ks, Ns, Es, TOPKs, DTYPEs, PF_TYPES, FE_TYPES, Q_TYPES)
|
||||
)
|
||||
|
||||
results_df: pd.DataFrame | None = None
|
||||
for m, k, n, e, topks, dtype, pf_type, experts_type, quant_config in tqdm(
|
||||
combinations
|
||||
):
|
||||
config = Config(
|
||||
Ms=[m],
|
||||
K=k,
|
||||
N=n,
|
||||
E=e,
|
||||
topks=topks,
|
||||
dtype=dtype,
|
||||
prepare_finalize_type=pf_type,
|
||||
fused_experts_type=experts_type,
|
||||
quant_config=quant_config,
|
||||
world_size=2,
|
||||
)
|
||||
|
||||
success = None
|
||||
if config.is_valid()[0]:
|
||||
print(f"Running config : {config.describe()} ...")
|
||||
try:
|
||||
weights: WeightTensors = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
rank_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
)
|
||||
success = Result.PASS
|
||||
except Exception as _:
|
||||
success = Result.FAIL
|
||||
else:
|
||||
success = Result.SKIP
|
||||
|
||||
results_df = add_to_results(config, success, results_df)
|
||||
|
||||
if results_df is not None:
|
||||
results_df.to_csv(f"{csv_file_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Make ModularKernel feature matrix \n"
|
||||
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.make_feature_matrix " # noqa: E501
|
||||
"-f ./feature_matrices/feature_matrix.csv"
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--feature-matrix-csv-file-path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="File name to Generate a .csv file",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
csv_path = args.feature_matrix_csv_file_path
|
||||
assert csv_path.endswith("csv"), (
|
||||
f"Need a file path ending with .csv, got {csv_path}"
|
||||
)
|
||||
assert Path(csv_path).parent.is_dir(), (
|
||||
f"Cannot find parent directory for {Path(csv_path).parent}"
|
||||
)
|
||||
|
||||
make_feature_matrix(args.feature_matrix_csv_file_path)
|
||||
@@ -1,182 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from .common import (
|
||||
Config,
|
||||
RankTensors,
|
||||
WeightTensors,
|
||||
_make_gscale,
|
||||
make_modular_kernel,
|
||||
)
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
|
||||
|
||||
|
||||
def do_profile(
|
||||
fn: Callable,
|
||||
fn_kwargs: dict[Any, Any],
|
||||
pgi: ProcessGroupInfo,
|
||||
config: Config,
|
||||
num_warmups: int = 5,
|
||||
):
|
||||
for _ in range(num_warmups):
|
||||
fn(**fn_kwargs)
|
||||
|
||||
with torch.profiler.profile(
|
||||
activities=[
|
||||
torch.profiler.ProfilerActivity.CPU,
|
||||
torch.profiler.ProfilerActivity.CUDA,
|
||||
],
|
||||
with_stack=True,
|
||||
record_shapes=True,
|
||||
) as tprof:
|
||||
fn(**fn_kwargs)
|
||||
device = torch.accelerator.current_device_index()
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
# TODO (varun): Add a descriptive trace file name
|
||||
tprof.export_chrome_trace(
|
||||
f"{config.torch_trace_dir_path}/m{config.M}_{pgi.rank}_trace.json"
|
||||
)
|
||||
|
||||
|
||||
def profile_modular_kernel(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
rank_tensors: RankTensors,
|
||||
) -> None:
|
||||
assert isinstance(config.Ms, int)
|
||||
assert isinstance(config.topks, int)
|
||||
|
||||
# weights for rank
|
||||
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
|
||||
|
||||
if config.quant_dtype == "nvfp4":
|
||||
gscale = _make_gscale(config.num_local_experts)
|
||||
else:
|
||||
gscale = None
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
config.quant_dtype,
|
||||
w1_scale=rank_weights.w1_scale,
|
||||
w2_scale=rank_weights.w2_scale,
|
||||
a1_scale=rank_tensors.hidden_states_scale,
|
||||
g1_alphas=(1 / rank_weights.w1_gs) if rank_weights.w1_gs is not None else None,
|
||||
g2_alphas=(1 / rank_weights.w2_gs) if rank_weights.w2_gs is not None else None,
|
||||
a1_gscale=gscale,
|
||||
a2_gscale=gscale,
|
||||
block_shape=config.quant_block_shape,
|
||||
per_act_token_quant=config.is_per_act_token_quant,
|
||||
per_out_ch_quant=config.is_per_out_ch_quant,
|
||||
)
|
||||
|
||||
# make modular kernel
|
||||
mk = make_modular_kernel(config, vllm_config, quant_config)
|
||||
|
||||
topk_ids = rank_tensors.topk_ids.to(
|
||||
mk.prepare_finalize.topk_indices_dtype() or rank_tensors.topk_ids.dtype
|
||||
)
|
||||
|
||||
# impls might update the tensor in place
|
||||
hidden_states = rank_tensors.hidden_states.clone()
|
||||
|
||||
mk_kwargs = {
|
||||
"hidden_states": hidden_states,
|
||||
"w1": rank_weights.w1,
|
||||
"w2": rank_weights.w2,
|
||||
"topk_weights": rank_tensors.topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"activation": MoEActivation.SILU,
|
||||
"expert_map": rank_tensors.expert_map,
|
||||
"global_num_experts": config.E,
|
||||
"apply_router_weight_on_input": config.topk == 1
|
||||
and config.supports_apply_weight_on_input(),
|
||||
}
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
num_tokens_across_dp = torch.tensor(
|
||||
[num_tokens] * config.world_size, device="cpu", dtype=torch.int
|
||||
)
|
||||
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
):
|
||||
do_profile(mk.apply, mk_kwargs, pgi, config)
|
||||
|
||||
|
||||
def rank_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
):
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
# workspace manager is normally initialized by GPUModelRunner; we initialize
|
||||
# it here for the standalone benchmark process.
|
||||
init_workspace_manager(torch.device(f"cuda:{pgi.local_rank}"))
|
||||
|
||||
# get weights to this device
|
||||
weights.to_current_device()
|
||||
|
||||
Ms = config.Ms
|
||||
assert isinstance(Ms, list)
|
||||
TOPKs = config.topks
|
||||
assert isinstance(TOPKs, list)
|
||||
|
||||
for m, topk in product(Ms, TOPKs):
|
||||
print(f"Running m={m}, topk={topk} ...")
|
||||
# override m and topk
|
||||
cfgx = copy.deepcopy(config)
|
||||
cfgx.Ms = m
|
||||
cfgx.topks = topk
|
||||
|
||||
# inputs for rank
|
||||
rank_tensors = RankTensors.make(cfgx, pgi)
|
||||
profile_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
|
||||
|
||||
|
||||
def run(config: Config):
|
||||
weights: WeightTensors = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
parallel_launch_with_config(
|
||||
config.world_size, rank_worker, vllm_config, env_dict, config, weights
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from .cli_args import make_config, make_config_arg_parser
|
||||
|
||||
parser = make_config_arg_parser(
|
||||
description=(
|
||||
"Run single prepare-finalize & fused-experts combination test"
|
||||
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.profile_modular_kernel " # noqa: E501
|
||||
"--pf-type DeepEPLLPrepareAndFinalize --experts-type BatchedTritonExperts"
|
||||
)
|
||||
)
|
||||
args = parser.parse_args()
|
||||
assert args.torch_trace_dir_path is not None, (
|
||||
"Please pass in a directory to store torch traces"
|
||||
)
|
||||
config = make_config(args)
|
||||
|
||||
run(config)
|
||||
@@ -1,259 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Platform-agnostic eligibility tests for GateLinear's ROCm router GEMMs.
|
||||
|
||||
These assert the dispatch flags directly, so they run device-free by mocking
|
||||
the platform predicates. ``allow_cublas_router_gemm`` selects the
|
||||
bf16xbf16->fp32 ``torch.mm`` epilogue, while ``allow_fp32_router_gemm`` selects
|
||||
the gfx950 low-M kernel with fp32 weights and output.
|
||||
|
||||
The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.router.gate_linear as gate_linear_mod
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
|
||||
def _make_gate(
|
||||
monkeypatch,
|
||||
*,
|
||||
is_rocm: bool,
|
||||
is_cuda: bool = False,
|
||||
bias: bool = False,
|
||||
params_dtype: torch.dtype = torch.bfloat16,
|
||||
out_dtype: torch.dtype | None = torch.float32,
|
||||
input_size: int = 2048,
|
||||
output_size: int = 64,
|
||||
on_gfx950: bool = False,
|
||||
parallel_world_size: int = 1,
|
||||
) -> GateLinear:
|
||||
"""Build a GateLinear with platform predicates mocked, no GPU needed."""
|
||||
for target in (
|
||||
"vllm.model_executor.layers.linear",
|
||||
"vllm.model_executor.parameter",
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
f"{target}.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"{target}.get_tensor_model_parallel_world_size",
|
||||
lambda: parallel_world_size,
|
||||
)
|
||||
|
||||
platform = gate_linear_mod.current_platform
|
||||
monkeypatch.setattr(platform, "is_cuda", lambda: is_cuda)
|
||||
monkeypatch.setattr(platform, "is_rocm", lambda: is_rocm)
|
||||
monkeypatch.setattr(platform, "is_device_capability", lambda *a, **k: False)
|
||||
monkeypatch.setattr(platform, "is_device_capability_family", lambda *a, **k: False)
|
||||
if is_rocm:
|
||||
import vllm.platforms.rocm as rocm_platform
|
||||
|
||||
monkeypatch.setattr(rocm_platform, "on_gfx950", lambda: on_gfx950)
|
||||
|
||||
return GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=bias,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_no_bias_bf16_fp32_enables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=False)
|
||||
assert not gate.allow_specialized_router_gemm
|
||||
assert gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bias_disables_fused_gemm(monkeypatch):
|
||||
# torch.mm cannot add a bias, so a biased gate must not take the fused path.
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=True)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_fp32_weight_disables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, params_dtype=torch.float32)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_non_fp32_out_dtype_disables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_non_rocm_non_cuda_disables_fused_gemm(monkeypatch):
|
||||
# Neither the CUDA specialized path nor the ROCm branch applies.
|
||||
gate = _make_gate(monkeypatch, is_rocm=False, is_cuda=False)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_set_out_dtype_enables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=False, out_dtype=None)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=True, out_dtype=None)
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size"),
|
||||
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
|
||||
)
|
||||
def test_rocm_gfx950_enables_fp32_router_gemm(
|
||||
monkeypatch, input_size: int, output_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=True,
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_size", [2, 4, 8])
|
||||
def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
|
||||
monkeypatch, ep_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=6144,
|
||||
output_size=128,
|
||||
on_gfx950=True,
|
||||
parallel_world_size=ep_size,
|
||||
)
|
||||
assert gate.weight.shape == (128, 6144)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
|
||||
[
|
||||
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
|
||||
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
|
||||
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
|
||||
],
|
||||
)
|
||||
def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
monkeypatch,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
bias: bool,
|
||||
on_gfx950: bool,
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=params_dtype,
|
||||
bias=bias,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROCm BF16x3 tier (fp32 router weights)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
|
||||
"""Build a gate with the ROCm bf16x3 platform check mocked."""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
|
||||
kwargs.setdefault("params_dtype", torch.float32)
|
||||
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
|
||||
|
||||
|
||||
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
# Built by process_weights_after_loading, not lazily in forward().
|
||||
# Registered up front but still None, so named_buffers() skips it.
|
||||
assert gate._bf16x3_weight is None
|
||||
assert "_bf16x3_weight" in gate._buffers
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
|
||||
# torch.mm has no bias term.
|
||||
gate = _make_bf16x3_gate(monkeypatch, bias=True)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
|
||||
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
|
||||
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
|
||||
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
|
||||
gate = _make_bf16x3_gate(monkeypatch, supported=False)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
assert not gate._rocm_bf16x3_weight_eligible
|
||||
|
||||
|
||||
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
|
||||
"""An ineligible out_dtype must not allocate a split that is never read;
|
||||
it is 1.5x the router weight on every layer."""
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
assert gate._bf16x3_weight is None
|
||||
|
||||
|
||||
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_dispatch_is_a_custom_op():
|
||||
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
|
||||
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
|
||||
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
|
||||
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
|
||||
assert out.shape == (8192, 128)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
|
||||
"""The gfx950 low-M fp32 tier claims these shapes too and returns
|
||||
unconditionally, so bf16x3 is only reachable if checked first."""
|
||||
gate = _make_bf16x3_gate(
|
||||
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
source = inspect.getsource(type(gate).forward)
|
||||
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
|
||||
"allow_fp32_router_gemm"
|
||||
)
|
||||
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
MODEL_PATH = "zai-org/chatglm3-6b"
|
||||
LORA_RANK = 64
|
||||
DEFAULT_MAX_LORAS = 4 * 3
|
||||
|
||||
|
||||
def get_lora_requests(lora_path) -> list[LoRARequest]:
|
||||
lora_requests: list[LoRARequest] = [
|
||||
LoRARequest(lora_name=f"{i}", lora_int_id=i, lora_path=lora_path)
|
||||
for i in range(1, DEFAULT_MAX_LORAS + 1)
|
||||
]
|
||||
return lora_requests
|
||||
|
||||
|
||||
async def requests_processing_time(llm, lora_requests: list[LoRARequest]) -> float:
|
||||
sampling_params = SamplingParams(
|
||||
n=1, temperature=0.0, top_p=1.0, ignore_eos=True, max_tokens=1
|
||||
)
|
||||
|
||||
generators = []
|
||||
start = time.perf_counter()
|
||||
|
||||
for lora_request in lora_requests:
|
||||
lora_int_id = lora_request.lora_int_id
|
||||
generator = llm.generate(
|
||||
prompt=TextPrompt(prompt=f"hello {lora_int_id}", multi_modal_data=None), # type: ignore
|
||||
sampling_params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
request_id=f"test{lora_int_id}",
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
all_gens = merge_async_iterators(*generators)
|
||||
async for i, res in all_gens:
|
||||
pass
|
||||
|
||||
end = time.perf_counter()
|
||||
return end - start
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_lora(chatglm3_lora_files):
|
||||
"""The add_lora function is used to preload some LoRA adapters into the
|
||||
engine in anticipation of future requests using these adapters. To test
|
||||
this functionality, we use the async engine to process some requests - We
|
||||
do it twice, once with add_lora() preloading and once without.
|
||||
|
||||
We measure the request processing time in both cases and expect the time
|
||||
to be lesser in the case with add_lora() calls.
|
||||
"""
|
||||
lora_requests: list[LoRARequest] = get_lora_requests(chatglm3_lora_files)
|
||||
|
||||
max_loras = len(set([lr.lora_int_id for lr in lora_requests]))
|
||||
# Create engine in eager-mode. Due to high max_loras, the CI can
|
||||
# OOM during cuda-graph capture.
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=MODEL_PATH,
|
||||
enable_lora=True,
|
||||
max_loras=max_loras,
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=128,
|
||||
gpu_memory_utilization=0.8, # avoid OOM
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
# split lora_requests into 3 parts
|
||||
part_size = len(lora_requests) // 3
|
||||
dummy_run_requests = lora_requests[:part_size]
|
||||
warmup_run_requests = lora_requests[part_size : part_size * 2]
|
||||
cold_run_requests = lora_requests[part_size * 2 :]
|
||||
|
||||
async with build_async_engine_client_from_engine_args(engine_args) as llm:
|
||||
# Dummy run - So any 1-time functionality like triton kernel compilation
|
||||
# is complete here.
|
||||
await requests_processing_time(llm, dummy_run_requests)
|
||||
|
||||
# Run with warmup
|
||||
add_lora_tasks = [llm.add_lora(lr) for lr in warmup_run_requests]
|
||||
add_lora_results = await asyncio.gather(*add_lora_tasks)
|
||||
|
||||
# Test that all all_lora calls are successful.
|
||||
assert all(add_lora_results)
|
||||
|
||||
time_with_add_lora = await requests_processing_time(llm, warmup_run_requests)
|
||||
|
||||
# Run without any warmup
|
||||
time_cold_start = await requests_processing_time(llm, cold_run_requests)
|
||||
|
||||
print(f"time hot-start {time_with_add_lora} vs time cold-start {time_cold_start} ")
|
||||
|
||||
assert time_with_add_lora < time_cold_start, (
|
||||
f"time_with_add_lora={time_with_add_lora}, "
|
||||
f"time_cold_start={time_cold_start}"
|
||||
"The engine request processing time with LoRA pre-loading "
|
||||
"must be less than the version that does on-demand LoRA loading."
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, EngineArgs
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.model_loader import tensorizer as tensorizer_mod
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
from vllm.v1.executor import UniProcExecutor
|
||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
MODEL_REF = "facebook/opt-125m"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model_ref():
|
||||
return MODEL_REF
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def allow_insecure_serialization(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup():
|
||||
cleanup_dist_env_and_memory(shutdown_ray=True)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def just_serialize_model_tensors(model_ref, monkeypatch, tmp_path):
|
||||
def noop(*args, **kwargs):
|
||||
return None
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tc = TensorizerConfig(tensorizer_uri=f"{tmp_path}/model.tensors")
|
||||
|
||||
monkeypatch.setattr(tensorizer_mod, "serialize_extra_artifacts", noop)
|
||||
|
||||
tensorizer_mod.tensorize_vllm_model(args, tc)
|
||||
yield tmp_path
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def tensorizer_config():
|
||||
config = TensorizerConfig(tensorizer_uri="vllm")
|
||||
return config
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model_path(model_ref, tmp_path):
|
||||
yield tmp_path / model_ref / "model.tensors"
|
||||
|
||||
|
||||
def assert_from_collective_rpc(engine: LLM, closure: Callable, closure_kwargs: dict):
|
||||
res = engine.collective_rpc(method=closure, kwargs=closure_kwargs)
|
||||
return all(res)
|
||||
|
||||
|
||||
# This is an object pulled from tests/v1/engine/test_engine_core.py
|
||||
# Modified to strip the `load_model` method from its `_init_executor`
|
||||
# method. It's purely used as a dummy utility to run methods that test
|
||||
# Tensorizer functionality
|
||||
class DummyExecutor(UniProcExecutor):
|
||||
def _init_executor(self) -> None:
|
||||
"""Initialize the worker and load the model."""
|
||||
self.driver_worker = WorkerWrapperBase(rpc_rank=0)
|
||||
distributed_init_method = get_file_store_init_method()
|
||||
local_rank = 0
|
||||
# set local rank as the device index if specified
|
||||
device_info = self.vllm_config.device_config.device.__str__().split(":")
|
||||
if len(device_info) > 1:
|
||||
local_rank = int(device_info[1])
|
||||
rank = 0
|
||||
is_driver_worker = True
|
||||
kwargs = dict(
|
||||
vllm_config=self.vllm_config,
|
||||
local_rank=local_rank,
|
||||
rank=rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
is_driver_worker=is_driver_worker,
|
||||
)
|
||||
self.mm_receiver_cache = None
|
||||
self.collective_rpc("init_worker", args=([kwargs],))
|
||||
self.collective_rpc("init_device")
|
||||
|
||||
def shutdown(self):
|
||||
if hasattr(self, "thread_pool"):
|
||||
self.thread_pool.shutdown(wait=False)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,17 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def require_rocm_cdna4() -> None:
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("requires ROCm")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("requires CDNA4")
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared BF16 baselines for the quantization comparison tests.
|
||||
|
||||
Several tests here quantize the same model and compare against an
|
||||
unquantized run of it. Computing that reference once per session avoids
|
||||
booting an identical engine for every quantization scheme under test.
|
||||
|
||||
Only the outputs are cached; each engine is still torn down as usual, so
|
||||
tests keep the isolation ``VllmRunner.__exit__`` provides.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def quant_baseline_logprobs(vllm_runner):
|
||||
"""Greedy logprobs for an unquantized model, computed once per config."""
|
||||
cache: dict = {}
|
||||
|
||||
def baseline(
|
||||
model: str,
|
||||
prompts: list[str],
|
||||
*,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
):
|
||||
key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts))
|
||||
if key not in cache:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=max_model_len,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
cache[key] = vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
return cache[key]
|
||||
|
||||
return baseline
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="vllm_add_dummy_model",
|
||||
version="0.1",
|
||||
packages=["vllm_add_dummy_model"],
|
||||
entry_points={
|
||||
"vllm.general_plugins": ["register_dummy_model = vllm_add_dummy_model:register"]
|
||||
},
|
||||
)
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="vllm_add_dummy_platform",
|
||||
version="0.1",
|
||||
packages=["vllm_add_dummy_platform"],
|
||||
entry_points={
|
||||
"vllm.platform_plugins": [
|
||||
"dummy_platform_plugin = vllm_add_dummy_platform:dummy_platform_plugin" # noqa
|
||||
],
|
||||
"vllm.general_plugins": [
|
||||
"dummy_custom_ops = vllm_add_dummy_platform:register_ops"
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
def dummy_platform_plugin() -> str | None:
|
||||
# Real platform plugins often import vLLM modules while registering.
|
||||
from vllm.utils.torch_utils import direct_register_custom_op # noqa: F401
|
||||
|
||||
return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
|
||||
|
||||
|
||||
def register_ops():
|
||||
import vllm_add_dummy_platform.dummy_custom_ops # noqa
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
|
||||
class DummyAttentionBackend(TritonAttentionBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "Dummy_Backend"
|
||||
@@ -1,32 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.platforms.interface import Platform, PlatformEnum
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
VllmConfig = None
|
||||
|
||||
|
||||
class DummyPlatform(Platform):
|
||||
_enum = PlatformEnum.OOT
|
||||
device_name = "DummyDevice"
|
||||
device_type: str = "privateuseone"
|
||||
dispatch_key: str = "PrivateUse1"
|
||||
|
||||
@classmethod
|
||||
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
|
||||
vllm_config.compilation_config.custom_ops = ["all"]
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
selected_backend: "AttentionBackendEnum",
|
||||
attn_selector_config: "AttentionSelectorConfig",
|
||||
num_heads: int | None = None,
|
||||
) -> str:
|
||||
return "vllm_add_dummy_platform.dummy_attention_backend.DummyAttentionBackend" # noqa E501
|
||||
@@ -1,30 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.v1.metrics.loggers import StatLoggerBase
|
||||
|
||||
|
||||
class DummyStatLogger(StatLoggerBase):
|
||||
"""A dummy stat logger for testing purposes.
|
||||
Implements the minimal interface expected by StatLoggerManager.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config, engine_idx=0):
|
||||
self.vllm_config = vllm_config
|
||||
self.engine_idx = engine_idx
|
||||
self.recorded = []
|
||||
self.logged = False
|
||||
self.engine_initialized = False
|
||||
|
||||
def record(
|
||||
self, scheduler_stats, iteration_stats, mm_cache_stats=None, engine_idx=0
|
||||
):
|
||||
self.recorded.append(
|
||||
(scheduler_stats, iteration_stats, mm_cache_stats, engine_idx)
|
||||
)
|
||||
|
||||
def log(self):
|
||||
self.logged = True
|
||||
|
||||
def log_engine_initialized(self):
|
||||
self.engine_initialized = True
|
||||
@@ -1,15 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="dummy_stat_logger",
|
||||
version="0.1",
|
||||
packages=["dummy_stat_logger"],
|
||||
entry_points={
|
||||
"vllm.stat_logger_plugins": [
|
||||
"dummy_stat_logger = dummy_stat_logger.dummy_stat_logger:DummyStatLogger" # noqa
|
||||
]
|
||||
},
|
||||
)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,36 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
|
||||
class DummyV1Scheduler(Scheduler):
|
||||
def schedule(self, throttle_prefills: bool = False):
|
||||
raise Exception("Exception raised by DummyV1Scheduler")
|
||||
|
||||
|
||||
def test_scheduler_plugins_v1(monkeypatch: pytest.MonkeyPatch):
|
||||
with monkeypatch.context() as m:
|
||||
# Explicitly turn off engine multiprocessing so
|
||||
# that the scheduler runs in this process
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
with pytest.raises(Exception) as exception_info:
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
enforce_eager=True, # reduce test time
|
||||
scheduler_cls=DummyV1Scheduler,
|
||||
)
|
||||
|
||||
engine = LLMEngine.from_engine_args(engine_args=engine_args)
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
engine.add_request("0", "foo", sampling_params)
|
||||
engine.step()
|
||||
|
||||
assert str(exception_info.value) == "Exception raised by DummyV1Scheduler"
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from dummy_stat_logger.dummy_stat_logger import DummyStatLogger
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.metrics.loggers import load_stat_logger_plugin_factories
|
||||
|
||||
|
||||
def test_stat_logger_plugin_is_discovered(monkeypatch: pytest.MonkeyPatch):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "dummy_stat_logger")
|
||||
|
||||
factories = load_stat_logger_plugin_factories()
|
||||
assert len(factories) == 1, f"Expected 1 factory, got {len(factories)}"
|
||||
assert factories[0] is DummyStatLogger, (
|
||||
f"Expected DummyStatLogger class, got {factories[0]}"
|
||||
)
|
||||
|
||||
# instantiate and confirm the right type
|
||||
vllm_config = VllmConfig()
|
||||
instance = factories[0](vllm_config)
|
||||
# The `is` check above narrows DummyStatLogger to a Callable
|
||||
assert isinstance(instance, DummyStatLogger)
|
||||
|
||||
|
||||
def test_no_plugins_loaded_if_env_empty(monkeypatch: pytest.MonkeyPatch):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "")
|
||||
|
||||
factories = load_stat_logger_plugin_factories()
|
||||
assert factories == []
|
||||
|
||||
|
||||
def test_invalid_stat_logger_plugin_raises(monkeypatch: pytest.MonkeyPatch):
|
||||
def fake_plugin_loader(group: str):
|
||||
assert group == "vllm.stat_logger_plugins"
|
||||
return {"bad": object()}
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"vllm.v1.metrics.loggers.load_plugins_by_group",
|
||||
fake_plugin_loader,
|
||||
)
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="Stat logger plugin 'bad' must be a subclass of StatLoggerBase",
|
||||
):
|
||||
load_stat_logger_plugin_factories()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_logger_plugin_integration_with_engine(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "dummy_stat_logger")
|
||||
|
||||
engine_args = AsyncEngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
enforce_eager=True, # reduce test time
|
||||
disable_log_stats=True, # disable default loggers
|
||||
)
|
||||
|
||||
engine = AsyncLLM.from_engine_args(engine_args=engine_args)
|
||||
|
||||
assert engine.logger_manager is not None
|
||||
assert len(engine.logger_manager.stat_loggers) == 2
|
||||
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 # type: ignore[attr-defined]
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], # type: ignore[attr-defined]
|
||||
DummyStatLogger,
|
||||
)
|
||||
|
||||
engine.shutdown()
|
||||
@@ -1,21 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def skip_unsupported_flashinfer_sampler():
|
||||
# CI runs this suite with FlashInfer both explicitly enabled and disabled.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and envs.is_set("VLLM_USE_FLASHINFER_SAMPLER")
|
||||
and envs.VLLM_USE_FLASHINFER_SAMPLER
|
||||
and current_platform.num_compute_units(torch.accelerator.current_device_index())
|
||||
<= 16
|
||||
):
|
||||
pytest.skip("FlashInfer top-k masking requires more than 16 SMs")
|
||||
@@ -109,45 +109,6 @@ def _write_json(path: Path, value: object) -> None:
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
|
||||
@pytest.mark.parametrize(
|
||||
("is_mm_prefix_lm", "is_multimodal_model", "expected"),
|
||||
[
|
||||
pytest.param(True, True, True, id="multimodal-prefix-lm"),
|
||||
pytest.param(False, True, False, id="multimodal-causal"),
|
||||
pytest.param(True, False, False, id="text-prefix-lm"),
|
||||
pytest.param(None, True, False, id="missing-model-config"),
|
||||
],
|
||||
)
|
||||
def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
|
||||
is_mm_prefix_lm: bool | None,
|
||||
is_multimodal_model: bool,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
config = SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(cudagraph_mode=CUDAGraphMode.NONE),
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
worker_cls="test-worker",
|
||||
),
|
||||
model_config=(
|
||||
None
|
||||
if is_mm_prefix_lm is None
|
||||
else SimpleNamespace(is_mm_prefix_lm=is_mm_prefix_lm)
|
||||
),
|
||||
scheduler_config=SimpleNamespace(
|
||||
is_multimodal_model=is_multimodal_model,
|
||||
disable_chunked_mm_input=False,
|
||||
),
|
||||
)
|
||||
|
||||
RocmPlatform.check_and_update_config(config)
|
||||
|
||||
assert config.scheduler_config.disable_chunked_mm_input is expected
|
||||
|
||||
|
||||
def _sampling_replay_config(
|
||||
*,
|
||||
return_sampling_mask: bool = True,
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# NOTE: Since CI runs the tests from the `tests` directory, it is necessary to rename
|
||||
# this module to avoid conflicting with HF's `tokenizers` package
|
||||
@@ -1,81 +0,0 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a specific location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city name"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "Temperature unit"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Search the web for information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query"
|
||||
},
|
||||
"num_results": {
|
||||
"type": "integer",
|
||||
"description": "Number of results to return"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in Beijing?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "The user wants to know the weather in Beijing. I should use the get_weather tool.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_001",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_001",
|
||||
"content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "Got the weather data. Let me format a nice response.",
|
||||
"content": "The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "The user said hello, I should greet back.",
|
||||
"content": "Hi there! How can I help you?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the capital of France?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "The user asks about the capital of France. It is Paris.",
|
||||
"content": "The capital of France is Paris."
|
||||
}
|
||||
]
|
||||
@@ -1,159 +0,0 @@
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "该助手为DeepSeek,由深度求索公司创造。"
|
||||
},
|
||||
{
|
||||
"role": "latest_reminder",
|
||||
"content": "2026-02-21,星期六,广州,App,中文"
|
||||
},
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "小柴胡冲剂和布洛芬能一起吃吗?\n\nCITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Web search. Split multiple queries with '||'.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"queries": {
|
||||
"type": "string",
|
||||
"description": "query1||query2"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"queries"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"description": "Batch open IDs (format 【{id}†...】) or URLs.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"open_list": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"description": "ID or URL",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"default": -1
|
||||
},
|
||||
"cursor": {
|
||||
"type": "integer",
|
||||
"description": "",
|
||||
"default": -1
|
||||
},
|
||||
"loc": {
|
||||
"type": "integer",
|
||||
"description": "Start line",
|
||||
"default": -1
|
||||
},
|
||||
"num_lines": {
|
||||
"type": "integer",
|
||||
"description": "",
|
||||
"default": -1
|
||||
},
|
||||
"view_source": {
|
||||
"type": "boolean",
|
||||
"description": "",
|
||||
"default": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"open_list"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"description": "Find exact text pattern in pages.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"find_list": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"cursor": {
|
||||
"type": "integer",
|
||||
"description": "",
|
||||
"default": -1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pattern"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"find_list"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning": "用户想知道小柴胡冲剂和布洛芬能否一起服用。",
|
||||
"tool_calls": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"queries\": \"小柴胡冲剂 布洛芬 相互作用 一起吃\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[0]"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "请及时就医。",
|
||||
"reasoning": "现在开始组织回答。",
|
||||
"tool_calls": []
|
||||
}
|
||||
]
|
||||
@@ -1,28 +0,0 @@
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "该助手为DeepSeek-V3,由深度求索公司创造。\n今天是2025年10月17日,星期五。"
|
||||
},
|
||||
{
|
||||
"role": "latest_reminder",
|
||||
"content": "2024-11-15,上海市,App,中文"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "热海大滚锅是世界著名温泉吗"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。\n\n为了让你快速了解,我整理了一个简要的表格:\n\n| 维度 | 热海大滚锅的地位与特点 |\n| :--- | :--- |\n| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |\n| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |\n| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |\n| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |\n\n### 💡 游玩攻略与温馨提示\n\n如果你计划前往热海大滚锅,这里有一些实用信息供你参考:\n\n- **门票与开放时间**:\n - **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。\n - **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。\n\n- **特色体验**:\n - **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。\n - **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。\n\n- **注意事项**:\n - **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。\n - **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。\n\n希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。",
|
||||
"mask": 1
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "世界著名温泉有哪些",
|
||||
"task": "action"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Search"
|
||||
}
|
||||
]
|
||||
@@ -1,39 +0,0 @@
|
||||
{
|
||||
"thinking_mode": "thinking",
|
||||
"reasoning_effort": "max",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful vision assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "请按“第一张、第二张”的顺序回答:第一张图"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "examples/images/carrots.jpeg"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "和第二张图"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "examples/images/corn.jpeg"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "中分别是什么食材?它们通常食用的部位分别是什么?"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
<|begin▁of▁sentence|>You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
|
||||
<|DSML|parameter name="unit" string="true">celsius</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -1 +0,0 @@
|
||||
<|begin▁of▁sentence|>You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -1,38 +0,0 @@
|
||||
<|begin▁of▁sentence|>该助手为DeepSeek,由深度求索公司创造。<|latest_reminder|>2026-02-21,星期六,广州,App,中文<|User|>小柴胡冲剂和布洛芬能一起吃吗?
|
||||
|
||||
CITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "search", "description": "Web search. Split multiple queries with '||'.", "parameters": {"type": "object", "properties": {"queries": {"type": "string", "description": "query1||query2"}}, "required": ["queries"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
|
||||
{"name": "open", "description": "Batch open IDs (format 【{id}†...】) or URLs.", "parameters": {"type": "object", "properties": {"open_list": {"type": "array", "items": {"type": "object", "properties": {"id": {"description": "ID or URL", "anyOf": [{"type": "integer"}, {"type": "string"}], "default": -1}, "cursor": {"type": "integer", "description": "", "default": -1}, "loc": {"type": "integer", "description": "Start line", "default": -1}, "num_lines": {"type": "integer", "description": "", "default": -1}, "view_source": {"type": "boolean", "description": "", "default": false}}, "additionalProperties": false}, "description": ""}}, "required": ["open_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
|
||||
{"name": "find", "description": "Find exact text pattern in pages.", "parameters": {"type": "object", "properties": {"find_list": {"type": "array", "items": {"type": "object", "properties": {"pattern": {"type": "string", "description": ""}, "cursor": {"type": "integer", "description": "", "default": -1}}, "required": ["pattern"], "additionalProperties": false}, "description": ""}}, "required": ["find_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|Assistant|><think>用户想知道小柴胡冲剂和布洛芬能否一起服用。</think>
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="search">
|
||||
<|DSML|parameter name="queries" string="true">小柴胡冲剂 布洛芬 相互作用 一起吃</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>[0]</tool_result><|Assistant|><think>现在开始组织回答。</think>请及时就医。<|end▁of▁sentence|>
|
||||
@@ -1,29 +0,0 @@
|
||||
<|begin▁of▁sentence|>该助手为DeepSeek-V3,由深度求索公司创造。
|
||||
今天是2025年10月17日,星期五。<|latest_reminder|>2024-11-15,上海市,App,中文<|User|>热海大滚锅是世界著名温泉吗<|Assistant|></think>关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。
|
||||
|
||||
为了让你快速了解,我整理了一个简要的表格:
|
||||
|
||||
| 维度 | 热海大滚锅的地位与特点 |
|
||||
| :--- | :--- |
|
||||
| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |
|
||||
| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |
|
||||
| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |
|
||||
| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |
|
||||
|
||||
### 💡 游玩攻略与温馨提示
|
||||
|
||||
如果你计划前往热海大滚锅,这里有一些实用信息供你参考:
|
||||
|
||||
- **门票与开放时间**:
|
||||
- **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。
|
||||
- **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。
|
||||
|
||||
- **特色体验**:
|
||||
- **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。
|
||||
- **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。
|
||||
|
||||
- **注意事项**:
|
||||
- **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。
|
||||
- **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。
|
||||
|
||||
希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。<|end▁of▁sentence|><|User|>世界著名温泉有哪些<|Assistant|></think><|action|>Search<|end▁of▁sentence|>
|
||||
@@ -1,9 +0,0 @@
|
||||
<|begin▁of▁sentence|>You are a helpful vision assistant.<|User|>请按“第一张、第二张”的顺序回答:第一张图
|
||||
|
||||
<|deepseek_image|>
|
||||
|
||||
和第二张图
|
||||
|
||||
<|deepseek_image|>
|
||||
|
||||
中分别是什么食材?它们通常食用的部位分别是什么?<|Assistant|></think>
|
||||
@@ -1,38 +0,0 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="get_weather">
|
||||
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
|
||||
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -1,3 +0,0 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
NUM_BLOCKS = 6
|
||||
MAX_MODEL_LEN = 1000
|
||||
LONG_PREFILL_THRESHOLD = 15
|
||||
EOS_TOKEN_ID = 50256
|
||||
|
||||
|
||||
def build_scheduler():
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-350m",
|
||||
trust_remote_code=True,
|
||||
dtype="float16",
|
||||
seed=42,
|
||||
skip_tokenizer_init=True,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
)
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=5,
|
||||
max_num_batched_tokens=200,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
long_prefill_token_threshold=LONG_PREFILL_THRESHOLD,
|
||||
enable_chunked_prefill=True,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
policy="priority",
|
||||
watermark=0.0,
|
||||
scheduler_reserve_full_isl=False,
|
||||
)
|
||||
cache_config = CacheConfig(
|
||||
block_size=BLOCK_SIZE,
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
enable_prefix_caching=False,
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(),
|
||||
)
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
cache_config.num_gpu_blocks = NUM_BLOCKS
|
||||
register_all_kvcache_specs(vllm_config)
|
||||
return Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
block_size=BLOCK_SIZE,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
)
|
||||
|
||||
|
||||
_hash_init = False
|
||||
|
||||
|
||||
def make_request(req_id, num_tokens, priority, arrival_time):
|
||||
global _hash_init
|
||||
if not _hash_init:
|
||||
init_none_hash(sha256)
|
||||
_hash_init = True
|
||||
block_hasher = get_request_block_hasher(BLOCK_SIZE, sha256)
|
||||
sp = SamplingParams(ignore_eos=True, max_tokens=50)
|
||||
sp.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
return Request(
|
||||
request_id=req_id,
|
||||
prompt_token_ids=[7] * num_tokens,
|
||||
sampling_params=sp,
|
||||
pooling_params=None,
|
||||
priority=priority,
|
||||
arrival_time=arrival_time,
|
||||
block_hasher=block_hasher,
|
||||
)
|
||||
|
||||
|
||||
def mock_output(out, tok=100):
|
||||
ids = list(out.num_scheduled_tokens.keys())
|
||||
return ModelRunnerOutput(
|
||||
req_ids=ids,
|
||||
req_id_to_index={r: i for i, r in enumerate(ids)},
|
||||
sampled_token_ids=[[tok] for _ in ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
|
||||
def test_priority_scheduler_preempt_skipped_request():
|
||||
scheduler = build_scheduler()
|
||||
|
||||
# R1: Add request A (Worst priority)
|
||||
A = make_request("A", num_tokens=200, priority=9, arrival_time=1.0)
|
||||
scheduler.add_request(A)
|
||||
out = scheduler.schedule()
|
||||
scheduler.update_from_output(out, mock_output(out))
|
||||
|
||||
# R2: Add request B (Best priority)
|
||||
B = make_request("B", num_tokens=15, priority=0, arrival_time=2.0)
|
||||
scheduler.add_request(B)
|
||||
out = scheduler.schedule()
|
||||
scheduler.update_from_output(out, mock_output(out))
|
||||
|
||||
# R3: Add request C (Middle priority)
|
||||
C = make_request("C", num_tokens=1, priority=1, arrival_time=3.0)
|
||||
scheduler.add_request(C)
|
||||
out = scheduler.schedule()
|
||||
scheduler.update_from_output(out, mock_output(out))
|
||||
|
||||
# R4: Trigger preemption
|
||||
out = scheduler.schedule(throttle_prefills=True)
|
||||
|
||||
is_c_scheduled = "C" in out.num_scheduled_tokens
|
||||
|
||||
assert is_c_scheduled, (
|
||||
"Bug present: Request C was silently skipped during scheduling "
|
||||
"because the req_index was not decremented after preempting A."
|
||||
)
|
||||
assert C.status == RequestStatus.RUNNING
|
||||
@@ -4,7 +4,6 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
@@ -13,7 +12,6 @@ import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
@@ -25,7 +23,6 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
@@ -185,7 +182,6 @@ class ExternalLBServerManager:
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
@@ -299,140 +295,6 @@ async def clients(servers: list[tuple[RemoteOpenAIServer, list[str]]]):
|
||||
]
|
||||
|
||||
|
||||
def _get_parallel_config(server: RemoteOpenAIServer):
|
||||
response = requests.get(server.url_for("server_info?config_format=json"))
|
||||
response.raise_for_status()
|
||||
|
||||
vllm_config = response.json()["vllm_config"]
|
||||
return vllm_config["parallel_config"]
|
||||
|
||||
|
||||
def _assert_expert_parallel_config(
|
||||
servers: list[tuple[RemoteOpenAIServer, list[str]]],
|
||||
):
|
||||
for server, server_args in servers:
|
||||
expected = "--enable-expert-parallel" in server_args
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
# Each request will hit one of the API servers
|
||||
# `n_reqs` is set so that there is a good chance each server
|
||||
# receives at least one request
|
||||
n_reqs = 2 * api_server_count * api_server_count
|
||||
parallel_configs = [_get_parallel_config(server) for _ in range(n_reqs)]
|
||||
api_process_counts = [c["_api_process_count"] for c in parallel_configs]
|
||||
api_process_ranks = [c["_api_process_rank"] for c in parallel_configs]
|
||||
|
||||
assert all(c == api_server_count for c in api_process_counts), (
|
||||
api_process_counts
|
||||
)
|
||||
assert all(0 <= r < api_server_count for r in api_process_ranks), (
|
||||
api_process_ranks
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,195 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
|
||||
# ruff: noqa: E501
|
||||
|
||||
|
||||
def test_fast_inc_detok_invalid_utf8_err_case():
|
||||
"""Test edge case where tokenizer can produce non-monotonic,
|
||||
invalid UTF-8 output, which breaks the internal state of
|
||||
tokenizers' DecodeStream.
|
||||
See https://github.com/vllm-project/vllm/issues/17448.
|
||||
|
||||
Thanks to reproducer from @fpaupier:
|
||||
https://gist.github.com/fpaupier/0ed1375bd7633c5be6c894b1c7ac1be3.
|
||||
"""
|
||||
tokenizer = AutoTokenizer.from_pretrained("google/gemma-3-1b-it")
|
||||
|
||||
# Create a test request
|
||||
prompt_token_ids = [107, 4606, 236787, 107]
|
||||
params = SamplingParams(skip_special_tokens=True)
|
||||
request = EngineCoreRequest(
|
||||
request_id="test",
|
||||
external_req_id="test-ext",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_features=None,
|
||||
sampling_params=params,
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
detokenizer = IncrementalDetokenizer.from_new_request(tokenizer, request)
|
||||
|
||||
assert detokenizer.__class__.__name__ == "FastIncrementalDetokenizer", (
|
||||
"Should use FastIncrementalDetokenizer by default"
|
||||
)
|
||||
|
||||
# Process tokens incrementally
|
||||
test_tokens = [
|
||||
236840,
|
||||
107,
|
||||
138,
|
||||
236782,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
6265,
|
||||
1083,
|
||||
623,
|
||||
121908,
|
||||
147418,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
6265,
|
||||
236779,
|
||||
2084,
|
||||
1083,
|
||||
623,
|
||||
203292,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
6265,
|
||||
236779,
|
||||
7777,
|
||||
1083,
|
||||
623,
|
||||
121908,
|
||||
147418,
|
||||
569,
|
||||
537,
|
||||
236789,
|
||||
65880,
|
||||
569,
|
||||
537,
|
||||
236789,
|
||||
62580,
|
||||
853,
|
||||
115693,
|
||||
210118,
|
||||
35178,
|
||||
16055,
|
||||
1270,
|
||||
759,
|
||||
215817,
|
||||
4758,
|
||||
1925,
|
||||
1117,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
5654,
|
||||
1083,
|
||||
623,
|
||||
110733,
|
||||
46291,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
5654,
|
||||
236779,
|
||||
2084,
|
||||
1083,
|
||||
623,
|
||||
136955,
|
||||
56731,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
5654,
|
||||
236779,
|
||||
7777,
|
||||
1083,
|
||||
623,
|
||||
194776,
|
||||
2947,
|
||||
496,
|
||||
109811,
|
||||
1608,
|
||||
890,
|
||||
215817,
|
||||
4758,
|
||||
1925,
|
||||
1117,
|
||||
2789,
|
||||
432,
|
||||
398,
|
||||
602,
|
||||
31118,
|
||||
569,
|
||||
124866,
|
||||
134772,
|
||||
509,
|
||||
19478,
|
||||
1640,
|
||||
33779,
|
||||
236743,
|
||||
236770,
|
||||
236819,
|
||||
236825,
|
||||
236771,
|
||||
432,
|
||||
398,
|
||||
432,
|
||||
237167,
|
||||
827,
|
||||
107,
|
||||
140,
|
||||
236775,
|
||||
77984,
|
||||
1083,
|
||||
623,
|
||||
2709,
|
||||
236745,
|
||||
2555,
|
||||
513,
|
||||
236789,
|
||||
602,
|
||||
31118,
|
||||
569,
|
||||
]
|
||||
|
||||
output = ""
|
||||
for i, token_id in enumerate(test_tokens):
|
||||
detokenizer.update([token_id], False)
|
||||
|
||||
finished = i == len(test_tokens) - 1
|
||||
output += detokenizer.get_next_output_text(finished, delta=True)
|
||||
|
||||
assert (
|
||||
output
|
||||
== r"""[
|
||||
{
|
||||
"source": "Résultats",
|
||||
"source_type": "CONCEPT",
|
||||
"source_description": "Résultats de l'analyse de l'impact des opérations israéliennes sur la frontière libanaise",
|
||||
"target": "Israël",
|
||||
"target_type": "ORGANIZATION",
|
||||
"target_description": "Pays qui a obtenu à sa frontière libanaise « un niveau de calme inédit depuis les années 1960 »",
|
||||
"relationship": "Obtention d'un niveau de"""
|
||||
)
|
||||
@@ -0,0 +1,552 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, count
|
||||
from typing import Any, Literal
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
DeviceConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
|
||||
ExampleConnector,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
from vllm.v1.core.sched.scheduler import Scheduler, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
EOS_TOKEN_ID = 50256
|
||||
|
||||
|
||||
def create_mock_connector() -> Mock:
|
||||
connector = Mock()
|
||||
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
|
||||
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
def maybe_update_block_size(block_size: int) -> int:
|
||||
return 64 if current_platform.is_xpu() else block_size
|
||||
|
||||
|
||||
def assert_scheduler_empty(scheduler: Scheduler):
|
||||
"""Confirm the scheduler is "empty" - i.e. no leaks."""
|
||||
# Scheduler Metadata.
|
||||
assert len(scheduler.requests) == 0
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.kv_holding_waiting) == 0
|
||||
assert not scheduler.deferred_waiting
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler.finished_req_ids) == 0
|
||||
assert len(scheduler.finished_recving_kv_req_ids) == 0
|
||||
assert len(scheduler._inflight_prefills) == 0
|
||||
assert not scheduler._kv_fetch_stages
|
||||
assert not any(scheduler._kv_fetch_counts.values())
|
||||
|
||||
# EncoderCacheManager.
|
||||
assert len(scheduler.encoder_cache_manager.freed) == 0
|
||||
assert len(scheduler.encoder_cache_manager.cached) == 0
|
||||
|
||||
# KVCache Manager.
|
||||
assert (
|
||||
len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].num_cached_block
|
||||
)
|
||||
== 0
|
||||
)
|
||||
num_free_blocks = (
|
||||
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
|
||||
)
|
||||
assert num_free_blocks == (scheduler.kv_cache_manager.block_pool.num_gpu_blocks - 1)
|
||||
|
||||
# NOTE(rob): just the ref count on blocks will be 0. The hash
|
||||
# value, etc will remain since we lazily evict for prefix cache.
|
||||
for block in scheduler.kv_cache_manager.block_pool.blocks:
|
||||
assert block.ref_cnt == 0
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 64,
|
||||
block_size: int = 16,
|
||||
max_model_len: int = 10000,
|
||||
enable_chunked_prefill: bool = True,
|
||||
enable_permute_local_kv: bool = False,
|
||||
kv_connector_extra_config: dict[str, Any] | None = None,
|
||||
dtype: str = "float16",
|
||||
cache_dtype: str = "auto",
|
||||
hf_overrides: dict[str, Any] | None = None,
|
||||
attention_backend: str | None = None,
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
|
||||
kv_connector: str = "NixlConnector",
|
||||
kv_connector_module_path: str | None = None,
|
||||
kv_role: str = "kv_consumer",
|
||||
disable_hybrid_kv_cache_manager: bool | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
) -> VllmConfig:
|
||||
"""Initialize VllmConfig For Testing."""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
trust_remote_code=True,
|
||||
dtype=dtype,
|
||||
seed=42,
|
||||
hf_overrides=hf_overrides or {},
|
||||
)
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
|
||||
)
|
||||
# Cache config, optionally force APC
|
||||
cache_config = CacheConfig(
|
||||
block_size=block_size,
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype=cache_dtype,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
# Connectors are constructed after layout resolution; mirror that here.
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=kv_connector,
|
||||
kv_connector_module_path=kv_connector_module_path,
|
||||
kv_role=kv_role,
|
||||
enable_permute_local_kv=enable_permute_local_kv,
|
||||
kv_connector_extra_config=kv_connector_extra_config or {},
|
||||
kv_load_failure_policy=kv_load_failure_policy,
|
||||
)
|
||||
attention_config = AttentionConfig(backend=attention_backend)
|
||||
speculative_config = (
|
||||
SpeculativeConfig(model="ngram", num_speculative_tokens=num_speculative_tokens)
|
||||
if num_speculative_tokens is not None
|
||||
else None
|
||||
)
|
||||
return VllmConfig(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
device_config=DeviceConfig("cpu"),
|
||||
attention_config=attention_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
def create_scheduler(
|
||||
vllm_config: VllmConfig,
|
||||
num_blocks: int = 10000,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Initialize Scheduler For Testing."""
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
if kv_cache_config is None:
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks, # A large number of blocks to hold all requests
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
vllm_config.cache_config.num_gpu_blocks = num_blocks
|
||||
|
||||
scheduler_cls = (
|
||||
AsyncScheduler if vllm_config.scheduler_config.async_scheduling else Scheduler
|
||||
)
|
||||
return scheduler_cls(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
|
||||
_request_count = count(1)
|
||||
_none_hash_initialized = False
|
||||
|
||||
|
||||
def create_request(
|
||||
request_id: int | None = None,
|
||||
num_tokens: int = 10,
|
||||
common_prefix_len=0,
|
||||
max_tokens: int = 16,
|
||||
do_remote_decode: bool = False,
|
||||
do_remote_prefill: bool = False,
|
||||
num_remote_blocks: int = 3,
|
||||
block_size: int = 16,
|
||||
hash_fn: Callable = sha256,
|
||||
) -> Request:
|
||||
"""Make dummy request for testing."""
|
||||
assert num_tokens >= common_prefix_len >= 0
|
||||
|
||||
if request_id is None:
|
||||
request_id = next(_request_count)
|
||||
|
||||
global _none_hash_initialized
|
||||
if not _none_hash_initialized:
|
||||
init_none_hash(hash_fn)
|
||||
_none_hash_initialized = True
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
if do_remote_decode:
|
||||
assert not do_remote_prefill
|
||||
kv_transfer_params = dict(do_remote_prefill=False, do_remote_decode=True)
|
||||
elif do_remote_prefill:
|
||||
kv_transfer_params = dict(
|
||||
do_remote_prefill=True,
|
||||
do_remote_decode=False,
|
||||
remote_engine_id="my-engine-id",
|
||||
remote_request_id=f"prefill-{request_id}",
|
||||
remote_block_ids=list(range(num_remote_blocks)),
|
||||
remote_host="my-host",
|
||||
remote_port=1234,
|
||||
tp_size=1,
|
||||
)
|
||||
|
||||
max_tokens = 1 if do_remote_decode else max_tokens
|
||||
sampling_params = SamplingParams(max_tokens=max_tokens)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
|
||||
common_prefix = [1] * common_prefix_len if common_prefix_len > 0 else []
|
||||
suffix = [i * request_id for i in range(num_tokens - common_prefix_len)]
|
||||
prompt_token_ids = common_prefix + suffix
|
||||
|
||||
req = Request(
|
||||
request_id=f"id-{request_id}",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(block_size, hash_fn),
|
||||
)
|
||||
req.kv_transfer_params = kv_transfer_params
|
||||
return req
|
||||
|
||||
|
||||
def create_model_runner_output(
|
||||
reqs: list[Request],
|
||||
finished_sending: set[str] | None = None,
|
||||
finished_recving: set[str] | None = None,
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
use_eos: bool = False,
|
||||
token_id: int = 0,
|
||||
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
|
||||
) -> ModelRunnerOutput:
|
||||
"""Make dummy model runner output for testing."""
|
||||
# Make request data.
|
||||
req_ids = [req.request_id for req in reqs]
|
||||
req_id_to_index = {req_id: idx for idx, req_id in enumerate(req_ids)}
|
||||
|
||||
# Make sampled tokens.
|
||||
sampled_token = EOS_TOKEN_ID if use_eos else token_id
|
||||
sampled_token_ids = [[sampled_token] for _ in req_ids]
|
||||
|
||||
kv_connector_output = (
|
||||
None
|
||||
if (
|
||||
finished_sending is None
|
||||
and finished_recving is None
|
||||
and invalid_block_ids is None
|
||||
and kv_connector_worker_meta is None
|
||||
)
|
||||
else KVConnectorOutput(
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
invalid_block_ids=invalid_block_ids or set(),
|
||||
kv_connector_worker_meta=kv_connector_worker_meta,
|
||||
)
|
||||
)
|
||||
|
||||
# Make output data structure.
|
||||
return ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index=req_id_to_index,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=None,
|
||||
kv_connector_output=kv_connector_output,
|
||||
)
|
||||
|
||||
|
||||
class TestExampleConnector(ExampleConnector):
|
||||
def __init__(
|
||||
self,
|
||||
config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
self.name = config.kv_transfer_config.kv_connector_extra_config["name"]
|
||||
self._connector = ExampleConnector(config, role, kv_cache_config)
|
||||
self.call_record: dict[str, int] = defaultdict(int)
|
||||
# Use a unique temp file per connector
|
||||
self._event_file = (
|
||||
tempfile.gettempdir()
|
||||
+ f"/connector_{self.name}-{self.role.name}_events.log"
|
||||
)
|
||||
# Start with an empty file
|
||||
with open(self._event_file, "w") as _:
|
||||
pass
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if name in (
|
||||
"_connector",
|
||||
"call_record",
|
||||
"name",
|
||||
"_event_file",
|
||||
"__class__",
|
||||
"__dict__",
|
||||
"__getattribute__",
|
||||
"__init__",
|
||||
): # avoid recursion
|
||||
return object.__getattribute__(self, name)
|
||||
if not hasattr(self._connector, name):
|
||||
return object.__getattribute__(self, name)
|
||||
attr = getattr(self._connector, name)
|
||||
|
||||
# Intercept calls to the connector interface and write an event
|
||||
# for each one to a file, which can be read back in the main test proc.
|
||||
if callable(attr):
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
self.call_record[name] += 1
|
||||
|
||||
# Include args that we're interested in
|
||||
to_log = [name]
|
||||
for arg in args:
|
||||
if isinstance(arg, int):
|
||||
to_log.append(str(arg))
|
||||
elif isinstance(arg, KVCacheBlocks):
|
||||
to_log.append(f"num_blocks={[len(b) for b in arg.blocks]}")
|
||||
|
||||
# Log the event as a line to the file
|
||||
try:
|
||||
with open(self._event_file, "a") as f:
|
||||
f.write(" ".join(to_log) + "\n")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Could not log event {name} for {self.name}: {e}")
|
||||
return attr(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
return attr
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MockKVConfig:
|
||||
matched_tokens: int = 0
|
||||
is_async: bool = False
|
||||
num_defers_before_matching: int = 0
|
||||
supports_divergent_local_hybrid_hits: bool = False
|
||||
|
||||
|
||||
class MockKVConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
# Scheduler tests check metadata.requests
|
||||
self.requests: list = []
|
||||
|
||||
|
||||
class MockKVConnector(KVConnectorBase_V1):
|
||||
"""Mock KV connector for scheduler tests, supporting both sync and async mode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
extra_config = self._kv_transfer_config.kv_connector_extra_config
|
||||
self.config = MockKVConfig(
|
||||
matched_tokens=extra_config["matched_tokens"],
|
||||
is_async=extra_config["is_async"],
|
||||
num_defers_before_matching=extra_config.get(
|
||||
"num_defers_before_matching", 0
|
||||
),
|
||||
supports_divergent_local_hybrid_hits=extra_config.get(
|
||||
"supports_divergent_local_hybrid_hits", False
|
||||
),
|
||||
)
|
||||
self._defers_left: defaultdict[str, int] = defaultdict(
|
||||
lambda: self.config.num_defers_before_matching
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return self.config.supports_divergent_local_hybrid_hits
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
if self._defers_left[request.request_id] > 0:
|
||||
self._defers_left[request.request_id] -= 1
|
||||
return (None, False)
|
||||
return (self.config.matched_tokens, self.config.is_async)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
pass
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
metadata = MockKVConnectorMetadata()
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for req_id in chain(
|
||||
(req.req_id for req in scheduler_output.scheduled_new_reqs),
|
||||
(
|
||||
req_id
|
||||
for req_id in cached_reqs.req_ids
|
||||
if req_id in cached_reqs.resumed_req_ids
|
||||
),
|
||||
):
|
||||
metadata.requests.append({"req_id": req_id})
|
||||
return metadata
|
||||
|
||||
def start_load_kv(self, kv_caches, finished_req_ids):
|
||||
pass
|
||||
|
||||
def wait_for_layer_load(self, layer_name):
|
||||
pass
|
||||
|
||||
def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs):
|
||||
pass
|
||||
|
||||
def wait_for_save(self):
|
||||
pass
|
||||
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"TestExampleConnector", __name__, TestExampleConnector.__name__
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MockKVConnector", __name__, MockKVConnector.__name__
|
||||
)
|
||||
|
||||
|
||||
def make_kv_cache_config(
|
||||
block_size: int,
|
||||
swa_enabled: bool = False,
|
||||
mamba_enabled: bool = False,
|
||||
sw_size: int = 128,
|
||||
num_blocks: int = 100,
|
||||
mamba_cache_mode: Literal["align", "none"] = "none",
|
||||
) -> KVCacheConfig:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer2"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
)
|
||||
]
|
||||
if swa_enabled:
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
["layer1", "layer3"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
sliding_window=sw_size,
|
||||
),
|
||||
)
|
||||
)
|
||||
if mamba_enabled:
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
["mamba0", "mamba1"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((16,), (16,)),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
),
|
||||
)
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
|
||||
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
|
||||
|
||||
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
|
||||
by the connector can be resolved to (addr, len, dev) byte ranges.
|
||||
"""
|
||||
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
|
||||
counts = strided_descs[:, 4].astype(np.intp)
|
||||
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
|
||||
out = np.repeat(strided_descs[:, :3], counts, axis=0)
|
||||
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
|
||||
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
|
||||
return out
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501
|
||||
DummyStatLogger,
|
||||
)
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM
|
||||
from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def log_stats_enabled_engine_args():
|
||||
"""Shared fixture providing common AsyncEngineArgs configuration
|
||||
used across multiple tests.
|
||||
"""
|
||||
return AsyncEngineArgs(
|
||||
model="distilbert/distilgpt2",
|
||||
dtype="half",
|
||||
disable_log_stats=False,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args):
|
||||
"""RayPrometheusStatLogger should replace the default PrometheusStatLogger."""
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger]
|
||||
)
|
||||
try:
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger
|
||||
)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args):
|
||||
"""It's still possible to use custom stat loggers exclusively by passing
|
||||
disable_log_stats=True in addition to a list of custom stat loggers.
|
||||
"""
|
||||
# Create engine_args with disable_log_stats=True for this test
|
||||
disabled_log_engine_args = copy.deepcopy(log_stats_enabled_engine_args)
|
||||
disabled_log_engine_args.disable_log_stats = True
|
||||
|
||||
# Disable default loggers; pass custom stat logger to the constructor
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
disabled_log_engine_args, stat_loggers=[DummyStatLogger]
|
||||
)
|
||||
|
||||
try:
|
||||
assert len(engine.logger_manager.stat_loggers) == 2
|
||||
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
|
||||
DummyStatLogger,
|
||||
)
|
||||
|
||||
# log_stats is still True, since custom stat loggers are used
|
||||
assert engine.log_stats
|
||||
finally:
|
||||
engine.shutdown()
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization
|
||||
)
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spawn_sitecustomize_factory(monkeypatch, tmp_path: Path):
|
||||
"""Return a function that installs a given sitecustomize payload.
|
||||
|
||||
Both ROCm and XPU force the `spawn` multiprocessing start method (see
|
||||
`vllm.platforms.rocm`/`vllm.platforms.xpu`), which re-imports modules in
|
||||
child processes instead of inheriting them via `fork`'s copy-on-write
|
||||
memory. A plain `monkeypatch.setattr` in the parent test process is
|
||||
therefore invisible to spawned EngineCore/worker subprocesses on these
|
||||
platforms, so the payload must be installed via `sitecustomize.py` on
|
||||
`PYTHONPATH` instead.
|
||||
"""
|
||||
if not (current_platform.is_rocm() or current_platform.is_xpu()):
|
||||
return lambda _: None
|
||||
|
||||
def install(lines: Iterable[str]) -> None:
|
||||
sc = tmp_path / "sitecustomize.py"
|
||||
sc.write_text("\n".join(lines) + "\n")
|
||||
monkeypatch.setenv(
|
||||
"PYTHONPATH",
|
||||
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
|
||||
)
|
||||
|
||||
return install
|
||||
@@ -1,6 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shutdown test utils."""
|
||||
|
||||
SHUTDOWN_TEST_TIMEOUT_SEC = 120
|
||||
SHUTDOWN_TEST_THRESHOLD_BYTES = 2 * 2**30
|
||||
@@ -1,611 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
XgrammarBackend,
|
||||
has_xgrammar_unsupported_json_features,
|
||||
validate_xgrammar_grammar,
|
||||
)
|
||||
from vllm.v1.structured_output.utils import choice_as_grammar
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def grammar_accepts(schema: dict, text: str) -> bool:
|
||||
return _is_grammar_accept_string(Grammar.from_json_schema(schema), text)
|
||||
|
||||
|
||||
# ================================================
|
||||
# Unsupported schemas
|
||||
# ================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_multipleOf_schemas():
|
||||
return [
|
||||
{"type": "number", "multipleOf": 120},
|
||||
{"Even": {"type": "number", "multipleOf": 2}},
|
||||
{"type": "integer", "multipleOf": 120},
|
||||
{"Even": {"type": "integer", "multipleOf": 2}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_array_schemas():
|
||||
return [
|
||||
# array + some constraints is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968
|
||||
{"type": "array", "uniqueItems": True},
|
||||
{"type": "array", "contains": {"type": "string"}},
|
||||
{"type": "array", "minContains": 1},
|
||||
{"type": "array", "maxContains": 5},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_string_schemas():
|
||||
return [
|
||||
# ========================================================
|
||||
# string + format is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967
|
||||
{"type": "string", "format": "non_existing_format"},
|
||||
# ========================================================
|
||||
#
|
||||
# ========================================================
|
||||
# string + format/pattern + length constraint is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966
|
||||
#
|
||||
# pattern/format is compiled but length bounds are silently dropped,
|
||||
# so the combination must be rejected instead of producing quietly
|
||||
# wrong output
|
||||
{"type": "string", "pattern": "^a+$", "maxLength": 2},
|
||||
{"type": "string", "pattern": "^a+$", "minLength": 3},
|
||||
{"type": "string", "format": "email", "maxLength": 10},
|
||||
# ========================================================
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_propertyNames_combinations():
|
||||
return [
|
||||
# ========================================================
|
||||
# propertyNames + patternProperties is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959
|
||||
#
|
||||
# xgrammar drops propertyNames whenever patternProperties is present,
|
||||
# so "grade_12" matches the pattern and escapes the name constraint
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
},
|
||||
# ========================================================
|
||||
#
|
||||
# ========================================================
|
||||
# propertyNames + maxLength is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
|
||||
},
|
||||
# ========================================================
|
||||
#
|
||||
# ========================================================
|
||||
# propertyNames + properties is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"required": ["Bad"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"enum": ["good"]},
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
},
|
||||
# ========================================================
|
||||
#
|
||||
# ========================================================
|
||||
# propertyNames + unevaluatedProperties is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961
|
||||
#
|
||||
# xgrammar's propertyNames-only branch falls back to an unconstrained
|
||||
# value type, so it drops unevaluatedProperties whether it restricts
|
||||
# values (a schema) or forbids them (false).
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": False,
|
||||
},
|
||||
# also keep in mind that unevaluatedProperties can be a schema
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": {"type": "integer"},
|
||||
},
|
||||
# ========================================================
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_patternProperties_combinations():
|
||||
return [
|
||||
# ========================================================
|
||||
# patternProperties + properties is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964
|
||||
#
|
||||
# JSON Schema requires a property matched by both `properties` and
|
||||
# `patternProperties` to satisfy both (conjunction), but xgrammar
|
||||
# compiles them as alternative branches, so satisfying either one is
|
||||
# enough.
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {"^x$": {"type": "integer"}},
|
||||
"properties": {"x": {"type": "string"}},
|
||||
},
|
||||
# ========================================================
|
||||
#
|
||||
# ========================================================
|
||||
# patternProperties + patternProperties is unsupported by xgrammar
|
||||
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965
|
||||
#
|
||||
# The same alternative-branches problem applies to overlapping
|
||||
# patternProperties patterns: a key matching both patterns only has to
|
||||
# satisfy one of them.
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"^a[a-z]*$": {"type": "string"},
|
||||
"^[a-z]*z$": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
# ========================================================
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_multibranch_allof():
|
||||
"""Regression tests for issue #56556
|
||||
[Bug]: JSON schema with multiple allOf branches is silently ignored
|
||||
by the xgrammar structured-output backend #56556
|
||||
|
||||
Xgrammar currently does NOT support multi-branch allOf.
|
||||
Bug reported to xgrammar team for tracking progress on resolving:
|
||||
https://github.com/mlc-ai/xgrammar/issues/937
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"enum": ["yes", "no"]},
|
||||
]
|
||||
},
|
||||
# Multi-branch test case with > 2 branches:
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"minLength": 2},
|
||||
{"maxLength": 10},
|
||||
]
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "integer"},
|
||||
{"minimum": 0},
|
||||
]
|
||||
},
|
||||
# Non root multi-branch allOf:
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"is_this_sparta": {
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"enum": ["yes", "no"]},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_vllm_issue_56556_schema():
|
||||
"""Bug repro from vLLM issue #56556"""
|
||||
return [
|
||||
{
|
||||
"$defs": {
|
||||
"Base": {
|
||||
"type": "object",
|
||||
"properties": {"x": {"type": "integer", "minimum": 10}},
|
||||
"required": ["x"],
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
{"$ref": "#/$defs/Base"},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"y": {"type": "string"}},
|
||||
"required": ["y"],
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ================================================
|
||||
# Supported schemas
|
||||
# ================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_frankenstein_schema():
|
||||
# IMPORTANT(arpera):
|
||||
# Do NOT add more keywords here! This schema is overcrowded enough that a new
|
||||
# keyword can end up having no effect on the compiled grammar while the
|
||||
# test still passes. Give a new keyword its own fixture and named test.
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"},
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"status": {"type": "string"},
|
||||
"scores": {"type": "array", "items": {"type": "number"}},
|
||||
"car_type": {"type": "string", "enum": ["sedan", "suv", "truck"]},
|
||||
"car_brand": {"type": "string", "pattern": "^[a-zA-Z]+$"},
|
||||
"short_description": {"type": "string", "maxLength": 50},
|
||||
"mileage": {"type": "number", "minimum": 0, "maximum": 1000000},
|
||||
"model_year": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 1900,
|
||||
"exclusiveMaximum": 2100,
|
||||
},
|
||||
"long_description": {"type": "string", "minLength": 50, "maxLength": 2000},
|
||||
"address": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"street": {"type": "string"},
|
||||
"city": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"minProperties": 1,
|
||||
"maxProperties": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pattern_properties_schema():
|
||||
return {
|
||||
"type": "object",
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_names_schema():
|
||||
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_names_with_additional_properties_schema():
|
||||
"""This combination propertyNames + additionalProperties
|
||||
was unsupported some time ago by xgrammar.
|
||||
So, this test case was in unsupported category before.
|
||||
But this xgrammar's PR
|
||||
https://github.com/mlc-ai/xgrammar/pull/836
|
||||
added support of propertyNames + additionalProperties
|
||||
This fix was released in xgrammar v0.2.7
|
||||
which vLLM pins in PR #57272
|
||||
"""
|
||||
return {
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z_]+$"},
|
||||
"additionalProperties": {"type": "integer"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_allof_anyof_and_oneof():
|
||||
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
|
||||
This test case was made during working on covering
|
||||
unsupported multi-branch allOf to make sure the bug
|
||||
is NOT present in other combinators.
|
||||
See for more context in this file test `unsupported_multibranch_allof`
|
||||
"""
|
||||
return [
|
||||
# single-branch allOf:
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
],
|
||||
},
|
||||
# Corner cases: empty allOf, this is a valid schema
|
||||
{"allOf": []},
|
||||
# multi-branch allOf:
|
||||
# is NOT supported yet, see vLLM issue #56556
|
||||
# single-branch anyOf:
|
||||
{
|
||||
"anyOf": [],
|
||||
},
|
||||
# multi-branch anyOf:
|
||||
{
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "integer"},
|
||||
],
|
||||
},
|
||||
# single-branch oneOf:
|
||||
{
|
||||
"oneOf": [
|
||||
{"type": "integer", "minimum": 0, "maximum": 100},
|
||||
],
|
||||
},
|
||||
# multi-branch oneOf:
|
||||
{
|
||||
"oneOf": [
|
||||
{"type": "integer", "minimum": 0, "maximum": 100},
|
||||
{"type": "string", "enum": ["auto", "none"]},
|
||||
],
|
||||
},
|
||||
# Corner case:
|
||||
# "allOf" is a property name, not allOf combinator keyword
|
||||
{"type": "object", "properties": {"allOf": {"type": "string"}}},
|
||||
]
|
||||
|
||||
|
||||
# ================================================
|
||||
# Test has_xgrammar_unsupported_json_features functionality
|
||||
# ================================================
|
||||
|
||||
|
||||
class TestHasXGrammarUnsupportedJsonFeatures:
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
"unsupported_string_schemas",
|
||||
"unsupported_multipleOf_schemas",
|
||||
"unsupported_array_schemas",
|
||||
"unsupported_propertyNames_combinations",
|
||||
"unsupported_patternProperties_combinations",
|
||||
# vLLM issue #56556
|
||||
"unsupported_multibranch_allof",
|
||||
"unsupported_vllm_issue_56556_schema",
|
||||
],
|
||||
)
|
||||
def test_unsupported_json_features(self, schema_type, request):
|
||||
schemas = request.getfixturevalue(schema_type)
|
||||
for schema in schemas:
|
||||
assert has_xgrammar_unsupported_json_features(schema), (
|
||||
f"Schema should be unsupported: {schema}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
"supported_frankenstein_schema",
|
||||
"pattern_properties_schema",
|
||||
"property_names_schema",
|
||||
"property_names_with_additional_properties_schema",
|
||||
# Additional test cases implemented during work on
|
||||
# vLLM issue #56556
|
||||
"supported_allof_anyof_and_oneof",
|
||||
],
|
||||
)
|
||||
def test_supported_json_features(self, schema_type, request):
|
||||
schemas = request.getfixturevalue(schema_type)
|
||||
if not isinstance(schemas, list):
|
||||
schemas = [schemas]
|
||||
for schema in schemas:
|
||||
assert not has_xgrammar_unsupported_json_features(schema), (
|
||||
f"Schema should be supported: {schema}"
|
||||
)
|
||||
|
||||
class TestPR48416Regressions:
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[
|
||||
{"type": ["number"], "multipleOf": 3},
|
||||
{
|
||||
"type": ["string", "null"],
|
||||
"pattern": "^a+$",
|
||||
"maxLength": 2,
|
||||
},
|
||||
{
|
||||
"type": ["array", "null"],
|
||||
"items": {"type": "integer"},
|
||||
"uniqueItems": True,
|
||||
},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_list_type_does_not_bypass_unsupported_feature_check(self, schema):
|
||||
assert has_xgrammar_unsupported_json_features(schema)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[
|
||||
{"type": ["string", "null"]},
|
||||
{"type": ["integer", "string"]},
|
||||
{"type": ["array", "null"], "items": {"type": "string"}},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"propertyNames": {"pattern": "^[a-z_]+$"},
|
||||
},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"patternProperties": {"^S": {"type": "string"}},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_supported_list_type_json_features(self, schema):
|
||||
assert not has_xgrammar_unsupported_json_features(schema)
|
||||
|
||||
|
||||
class TestIsGrammarAcceptString:
|
||||
class TestPR42904Support:
|
||||
def test_property_names_constrains_keys(self, property_names_schema):
|
||||
assert grammar_accepts(property_names_schema, '{"score": 5}')
|
||||
assert grammar_accepts(property_names_schema, '{"score": "seven"}')
|
||||
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
|
||||
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
|
||||
|
||||
def test_property_names_keeps_additional_properties_value_schema(
|
||||
self, property_names_with_additional_properties_schema
|
||||
):
|
||||
schema = property_names_with_additional_properties_schema
|
||||
assert grammar_accepts(schema, '{"score": 5}')
|
||||
assert not grammar_accepts(schema, '{"Score": 5}')
|
||||
assert not grammar_accepts(schema, '{"score": "seven"}')
|
||||
|
||||
def test_pattern_properties_constrains_keys_and_values(
|
||||
self, pattern_properties_schema
|
||||
):
|
||||
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
|
||||
|
||||
class TestPR48115Regressions:
|
||||
@pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F])
|
||||
def test_choice_as_grammar_preserves_control_characters(self, codepoint):
|
||||
choice = f"a{chr(codepoint)}f"
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar([choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, "a")
|
||||
assert not _is_grammar_accept_string(grammar, "af")
|
||||
assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("choice", "other"),
|
||||
[
|
||||
(r"a\nf", "a\nf"),
|
||||
('a quote " and a backslash \\', 'a quote " and a backslash '),
|
||||
("café", "cafe"),
|
||||
("日本語", "日本"),
|
||||
("😀", "😁"),
|
||||
],
|
||||
)
|
||||
def test_choice_as_grammar_preserves_literal_choices(self, choice, other):
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, other)
|
||||
assert not _is_grammar_accept_string(grammar, choice + "extra")
|
||||
|
||||
def test_validate_xgrammar_preserves_multiline_choice(self):
|
||||
structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"])
|
||||
validate_xgrammar_grammar(
|
||||
SamplingParams(structured_outputs=structured_outputs)
|
||||
)
|
||||
|
||||
assert structured_outputs.choice is None
|
||||
grammar = Grammar.from_ebnf(structured_outputs.grammar)
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, "no\nplease")
|
||||
assert not _is_grammar_accept_string(grammar, r"no\nplease")
|
||||
|
||||
|
||||
# ================================================
|
||||
# Test XgrammarBackend functionality
|
||||
# ================================================
|
||||
|
||||
|
||||
class TestXgrammarBackend:
|
||||
class TestPR58067Regressions:
|
||||
"""Here we check behavior logic of our flag
|
||||
disable_any_whitespace when used with xgrammar backend.
|
||||
"""
|
||||
|
||||
_TOKENIZER = "openai-community/gpt2"
|
||||
_VOCAB_SIZE = 50257
|
||||
tokenizer: ClassVar[PreTrainedTokenizerBase]
|
||||
|
||||
@classmethod
|
||||
@pytest.fixture(scope="class", autouse=True)
|
||||
def _shared_tokenizer(cls):
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(cls._TOKENIZER)
|
||||
|
||||
@classmethod
|
||||
def _backend(cls, *, disable_any_whitespace: bool) -> XgrammarBackend:
|
||||
vllm_config = VllmConfig(
|
||||
structured_outputs_config=StructuredOutputsConfig(
|
||||
backend="xgrammar",
|
||||
disable_any_whitespace=disable_any_whitespace,
|
||||
)
|
||||
)
|
||||
return XgrammarBackend(
|
||||
vllm_config, tokenizer=cls.tokenizer, vocab_size=cls._VOCAB_SIZE
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _accepts_json(cls, backend: XgrammarBackend, json_text: str) -> bool:
|
||||
grammar = backend.compile_grammar(StructuredOutputOptions.JSON_OBJECT, "")
|
||||
return grammar.accept_tokens("req", cls.tokenizer.encode(json_text))
|
||||
|
||||
def test_disable_any_whitespace_is_false(self):
|
||||
"""Verify expected behavior of disable_any_whitespace=False"""
|
||||
backend = self._backend(disable_any_whitespace=False)
|
||||
|
||||
# Check that grammar accepts json with NO space after colon
|
||||
assert self._accepts_json(backend, '{"no_space_after_me":"yes_ofcourse"}')
|
||||
|
||||
# Check that grammar accepts json with space after colon
|
||||
assert self._accepts_json(backend, '{"yes_space_after_me": true}')
|
||||
|
||||
# Check that grammar accepts json with NO space after comma
|
||||
assert self._accepts_json(
|
||||
backend,
|
||||
'{"never_space_after_comma":"accepted","good_boy":":3"}',
|
||||
)
|
||||
|
||||
# Check that grammar accepts json with space after comma
|
||||
assert self._accepts_json(
|
||||
backend,
|
||||
'{"where_space_after_comma":42, "wow_thats_cool":4242}',
|
||||
)
|
||||
|
||||
def test_disable_any_whitespace_is_true(self):
|
||||
"""Verify expected behavior of disable_any_whitespace=True"""
|
||||
backend = self._backend(disable_any_whitespace=True)
|
||||
|
||||
# Check that grammar accepts json with NO space after colon
|
||||
assert self._accepts_json(backend, '{"no_space_after_me":"yes_ofcourse"}')
|
||||
|
||||
# Check that grammar rejects json with space after colon
|
||||
assert not self._accepts_json(backend, '{"yes_space_after_me": true}')
|
||||
|
||||
# FIXME(arpera):
|
||||
# Comma + separators=(",", ":") is wrong on xgrammar <=0.2.8
|
||||
# Seehttps://github.com/mlc-ai/xgrammar/issues/945
|
||||
# Re-enable both tests once vLLM pins xgrammar with the fix (>=0.2.9).
|
||||
#
|
||||
# # Check that grammar accepts json with NO space after comma
|
||||
# assert self._accepts_json(
|
||||
# backend,
|
||||
# '{"never_space_after_comma":"accepted","good_boy":":3"}',
|
||||
# )
|
||||
#
|
||||
# # Check that grammar rejects json with space after comma
|
||||
# assert not self._accepts_json(
|
||||
# backend,
|
||||
# '{"where_space_after_comma":42, "wow_thats_cool":4242}',
|
||||
# )
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user