[V1] Logits processors extensibility (#19912)
Signed-off-by: Andrew Feldman <afeldman@redhat.com> Signed-off-by: Andrew Feldman <afeld2012@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com> Co-authored-by: Andrew Feldman <afeld2012@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
Andrew Feldman
Harry Mellor
parent
4fc722eca4
commit
bf7f470b22
+66
-13
@@ -13,6 +13,7 @@ import tempfile
|
||||
import time
|
||||
import warnings
|
||||
from contextlib import contextmanager, suppress
|
||||
from multiprocessing import Process
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Literal, Optional, Union
|
||||
|
||||
@@ -76,6 +77,23 @@ VLLM_PATH = Path(__file__).parent.parent
|
||||
class RemoteOpenAIServer:
|
||||
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
|
||||
|
||||
def _start_server(self, model: str, vllm_serve_args: list[str],
|
||||
env_dict: Optional[dict[str, str]]) -> None:
|
||||
"""Subclasses override this method to customize server process launch
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
# the current process might initialize cuda,
|
||||
# to be safe, we should use spawn method
|
||||
env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
self.proc: subprocess.Popen = subprocess.Popen(
|
||||
["vllm", "serve", model, *vllm_serve_args],
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
)
|
||||
|
||||
def __init__(self,
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
@@ -128,18 +146,7 @@ class RemoteOpenAIServer:
|
||||
model_loader = get_model_loader(load_config)
|
||||
model_loader.download_model(model_config)
|
||||
|
||||
env = os.environ.copy()
|
||||
# the current process might initialize cuda,
|
||||
# to be safe, we should use spawn method
|
||||
env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
self.proc = subprocess.Popen(
|
||||
["vllm", "serve", model, *vllm_serve_args],
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
)
|
||||
self._start_server(model, vllm_serve_args, env_dict)
|
||||
max_wait_seconds = max_wait_seconds or 240
|
||||
self._wait_for_server(url=self.url_for("health"),
|
||||
timeout=max_wait_seconds)
|
||||
@@ -155,6 +162,10 @@ class RemoteOpenAIServer:
|
||||
# force kill if needed
|
||||
self.proc.kill()
|
||||
|
||||
def _poll(self) -> Optional[int]:
|
||||
"""Subclasses override this method to customize process polling"""
|
||||
return self.proc.poll()
|
||||
|
||||
def _wait_for_server(self, *, url: str, timeout: float):
|
||||
# run health check
|
||||
start = time.time()
|
||||
@@ -169,7 +180,7 @@ class RemoteOpenAIServer:
|
||||
# which means the server is not ready yet.
|
||||
# the stack trace is not useful, so we suppress it
|
||||
# by using `raise from None`.
|
||||
result = self.proc.poll()
|
||||
result = self._poll()
|
||||
if result is not None and result != 0:
|
||||
raise RuntimeError("Server exited unexpectedly.") from None
|
||||
|
||||
@@ -205,6 +216,48 @@ class RemoteOpenAIServer:
|
||||
**kwargs)
|
||||
|
||||
|
||||
class RemoteOpenAIServerCustom(RemoteOpenAIServer):
|
||||
"""Launch test server with custom child process"""
|
||||
|
||||
def _start_server(self, model: str, vllm_serve_args: list[str],
|
||||
env_dict: Optional[dict[str, str]]) -> None:
|
||||
self.proc: Process = Process(
|
||||
target=self.child_process_fxn,
|
||||
args=(env_dict, model,
|
||||
vllm_serve_args)) # type: ignore[assignment]
|
||||
self.proc.start()
|
||||
|
||||
def __init__(self,
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
child_process_fxn: Callable[
|
||||
[Optional[dict[str, str]], str, list[str]], None],
|
||||
*,
|
||||
env_dict: Optional[dict[str, str]] = None,
|
||||
seed: Optional[int] = 0,
|
||||
auto_port: bool = True,
|
||||
max_wait_seconds: Optional[float] = None) -> None:
|
||||
"""Store custom child process function then invoke superclass
|
||||
constructor which will indirectly launch it."""
|
||||
self.child_process_fxn = child_process_fxn
|
||||
super().__init__(model=model,
|
||||
vllm_serve_args=vllm_serve_args,
|
||||
env_dict=env_dict,
|
||||
seed=seed,
|
||||
auto_port=auto_port,
|
||||
max_wait_seconds=max_wait_seconds)
|
||||
|
||||
def _poll(self) -> Optional[int]:
|
||||
return self.proc.exitcode
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.proc.terminate()
|
||||
self.proc.join(8)
|
||||
if self.proc.is_alive():
|
||||
# force kill if needed
|
||||
self.proc.kill()
|
||||
|
||||
|
||||
def _test_completion(
|
||||
client: openai.OpenAI,
|
||||
model: str,
|
||||
|
||||
+16
-8
@@ -9,11 +9,13 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from tests.v1.sample.utils import (LogitsprocsTestFakes, create_fake_logits,
|
||||
create_penalty_tensor,
|
||||
create_prompt_tokens_tensor,
|
||||
fake_apply_logitsprocs,
|
||||
fake_update_logitsprocs_state)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import is_pin_memory_available
|
||||
@@ -24,7 +26,7 @@ from vllm.v1.sample.logits_processor import (BatchUpdate, BatchUpdateBuilder,
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
MoveDirectionality,
|
||||
init_builtin_logitsprocs)
|
||||
build_logitsprocs)
|
||||
# yapf: enable
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
@@ -53,6 +55,7 @@ class LogitsProcsRequestParams:
|
||||
workload_index: int
|
||||
logitproc_type: LogitprocType # Logitproc enabled, specified by str id
|
||||
out_tokens: list[int] # Output tokens required for min tokens test
|
||||
prompt_tokens: list[int] # Dummy prompt tokens placeholder
|
||||
params: SamplingParams # Settings customized for logitproc
|
||||
|
||||
def __init__(self, workload_index: int, logitproc_type: LogitprocType):
|
||||
@@ -63,6 +66,7 @@ class LogitsProcsRequestParams:
|
||||
# don't matter *for these tests* so use 0 as a dummy value
|
||||
self.out_tokens = ([0] *
|
||||
(MIN_TOKENS_LEN_THRESHOLD * random.randint(0, 2)))
|
||||
self.prompt_tokens = []
|
||||
self.params = _sampling_params_from_logitproc(logitproc_type)
|
||||
|
||||
def __str__(self):
|
||||
@@ -88,11 +92,12 @@ def _generate_fake_sampling_metadata(
|
||||
vocab_size,
|
||||
size=np.random.randint(
|
||||
1, MAX_NUM_PROMPT_TOKENS)).tolist())
|
||||
logitsprocs = init_builtin_logitsprocs(
|
||||
pin_memory_available=PIN_MEMORY_AVAILABLE,
|
||||
max_num_reqs=MAX_NUM_REQS + 1,
|
||||
device=device)
|
||||
|
||||
logitsprocs = build_logitsprocs(
|
||||
vllm_config=VllmConfig(),
|
||||
device=device,
|
||||
is_pin_memory=PIN_MEMORY_AVAILABLE,
|
||||
is_pooling_model=False,
|
||||
)
|
||||
fake_sampling_metadata = SamplingMetadata(
|
||||
temperature=torch.full((batch_size, ), 0.0),
|
||||
all_greedy=True,
|
||||
@@ -462,7 +467,8 @@ def _generate_fake_step_update(
|
||||
# Replace as many removed requests as possible with added requests
|
||||
add_remove_idx = batch_update_builder.pop_removed()
|
||||
batch_update_builder.added.append(
|
||||
(add_remove_idx, add_req_params.params, add_req_params.out_tokens))
|
||||
(add_remove_idx, add_req_params.params,
|
||||
add_req_params.prompt_tokens, add_req_params.out_tokens))
|
||||
persistent_batch[add_remove_idx] = add_req_params
|
||||
|
||||
# Append remaining added requests to end of batch
|
||||
@@ -470,7 +476,8 @@ def _generate_fake_step_update(
|
||||
num_step_add_replace):(wdx +
|
||||
num_step_add)]
|
||||
batch_update_builder.added.extend([
|
||||
(adx + batch_size, add_req_params.params, add_req_params.out_tokens)
|
||||
(adx + batch_size, add_req_params.params, add_req_params.prompt_tokens,
|
||||
add_req_params.out_tokens)
|
||||
for adx, add_req_params in enumerate(add_reqs_append)
|
||||
])
|
||||
persistent_batch.extend(add_reqs_append)
|
||||
@@ -561,6 +568,7 @@ def _assert_valid(
|
||||
step_idx=step_idx)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("reqs_per_logitproc", [REQS_PER_LOGITPROC])
|
||||
@pytest.mark.parametrize("logitsprocs_under_test", _get_test_cases())
|
||||
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
import sys
|
||||
from typing import Union
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
# yapf: disable
|
||||
from tests.v1.logits_processors.utils import (DUMMY_LOGITPROC_ARG,
|
||||
DUMMY_LOGITPROC_FQCN,
|
||||
DUMMY_LOGITPROC_MODULE,
|
||||
MAX_TOKENS, MODEL_NAME,
|
||||
POOLING_MODEL_NAME, TEMP_GREEDY,
|
||||
CustomLogitprocSource,
|
||||
DummyLogitsProcessor,
|
||||
dummy_module)
|
||||
from tests.v1.logits_processors.utils import entry_points as fake_entry_points
|
||||
from tests.v1.logits_processors.utils import prompts
|
||||
# yapf: enable
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.v1.sample.logits_processor import (STR_POOLING_REJECTS_LOGITSPROCS,
|
||||
LogitsProcessor)
|
||||
|
||||
# Create a mixture of requests which do and don't utilize the dummy logitproc
|
||||
sampling_params_list = [
|
||||
SamplingParams(temperature=TEMP_GREEDY,
|
||||
max_tokens=MAX_TOKENS,
|
||||
extra_args={DUMMY_LOGITPROC_ARG: 128}),
|
||||
SamplingParams(temperature=TEMP_GREEDY, max_tokens=MAX_TOKENS),
|
||||
SamplingParams(temperature=TEMP_GREEDY,
|
||||
max_tokens=MAX_TOKENS,
|
||||
extra_args={DUMMY_LOGITPROC_ARG: 67}),
|
||||
SamplingParams(temperature=TEMP_GREEDY, max_tokens=MAX_TOKENS),
|
||||
]
|
||||
|
||||
|
||||
def _run_test(kwargs: dict, logitproc_loaded: bool) -> None:
|
||||
"""Compare `LLM` instance initialized with specified `kwargs` against
|
||||
reference `LLM` instance.
|
||||
|
||||
Two scenarios:
|
||||
1. Server has loaded dummy logitproc; test that requests which specify
|
||||
dummy logitproc arg value behave as if logitproc is operating (output
|
||||
token value should repeat), while requests that don't specify dummy
|
||||
logitproc arg value should match reference `LLM` output.
|
||||
2. Server has *not* loaded dummy logitproc; test that all requests
|
||||
behave as if logitproc is *not* operating (output matches reference
|
||||
`LLM` output.)
|
||||
|
||||
Args:
|
||||
kwargs: `LLM` constructor kwargs
|
||||
logitproc_loaded: server has loaded dummy logitproc if True
|
||||
"""
|
||||
|
||||
# Create a vLLM instance and load custom logitproc
|
||||
llm_logitproc = LLM(
|
||||
model=MODEL_NAME,
|
||||
gpu_memory_utilization=0.1,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Create a reference vLLM instance without custom logitproc
|
||||
llm_ref = LLM(model=MODEL_NAME, gpu_memory_utilization=0.1)
|
||||
|
||||
# Run inference with logitproc loaded
|
||||
outputs_logitproc = llm_logitproc.generate(prompts, sampling_params_list)
|
||||
|
||||
# Reference run
|
||||
outputs_ref = llm_ref.generate(prompts, sampling_params_list)
|
||||
|
||||
# Validate outputs
|
||||
for bdx, (out_lp, out_ref, params) in enumerate(
|
||||
zip(outputs_logitproc, outputs_ref, sampling_params_list)):
|
||||
lp_toks = out_lp.outputs[0].token_ids
|
||||
if logitproc_loaded and params.extra_args:
|
||||
# This request exercises custom logitproc; validate that logitproc
|
||||
# forces `target_token` to be decoded in each step
|
||||
target_token = params.extra_args[DUMMY_LOGITPROC_ARG]
|
||||
if not all(x == target_token for x in lp_toks):
|
||||
raise AssertionError(
|
||||
f"Request {bdx} generated {lp_toks}, shoud all be "
|
||||
f"{target_token}")
|
||||
else:
|
||||
# This request does not exercise custom logitproc (or custom
|
||||
# logitproc is not enabled on this server); validate against
|
||||
# reference result
|
||||
ref_toks = out_ref.outputs[0].token_ids
|
||||
if lp_toks != ref_toks:
|
||||
raise AssertionError(
|
||||
f"Request {bdx} generated {lp_toks}, should match "
|
||||
f"{ref_toks}")
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("logitproc_source", list(CustomLogitprocSource))
|
||||
def test_custom_logitsprocs(monkeypatch,
|
||||
logitproc_source: CustomLogitprocSource):
|
||||
"""Test offline Python interface for passing custom logitsprocs
|
||||
|
||||
Construct an `LLM` instance which loads a custom logitproc that has a
|
||||
well-defined behavior (mask out all tokens except one `target_token`)
|
||||
|
||||
Construct a reference `LLM` instance with no custom logitproc
|
||||
|
||||
Pass in a batch of requests, 50% of which pass a `target_token` value
|
||||
in through `SamplingParams.extra_args`, 50% of which do not.
|
||||
|
||||
Validate that
|
||||
* Requests which do not activate the custom logitproc, yield the same
|
||||
results for both `LLM` instances
|
||||
* Requests which activate the custom logitproc, only output `target_token`
|
||||
|
||||
Test four scenarios, corresponding to `logitproc_source` value
|
||||
* No logitsprocs loaded - test that generated tokens match reference `LLM`
|
||||
instance output
|
||||
* Logitproc passed in via {entrypoint, class object, fully-qualified class
|
||||
name (FQCN)} - test that dummy logitproc is utilized correctly when
|
||||
provided via any of these three possible sources
|
||||
|
||||
Args:
|
||||
monkeypatch: for setting env vars
|
||||
logitproc_source: what source (entrypoint, fully-qualified class name
|
||||
(FQCN), class object, or None) the user pulls the
|
||||
logitproc from
|
||||
"""
|
||||
|
||||
# Test that logitproc info is passed to workers
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
|
||||
random.seed(40)
|
||||
|
||||
# Choose LLM args based on logitproc source
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_NONE:
|
||||
# Scenario: the server does not load any custom logitproc
|
||||
# Every other scenario is a different way of loading a custom logitproc
|
||||
_run_test({}, logitproc_loaded=False)
|
||||
return
|
||||
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_ENTRYPOINT:
|
||||
# Scenario: vLLM loads a logitproc from a preconfigured entrypoint
|
||||
# To that end, mock a dummy logitproc entrypoint
|
||||
import importlib.metadata
|
||||
importlib.metadata.entry_points = fake_entry_points # type: ignore
|
||||
|
||||
# fork is required for workers to see entrypoint patch
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "fork")
|
||||
_run_test({}, logitproc_loaded=True)
|
||||
return
|
||||
|
||||
kwargs: dict[str, list[Union[str, type[LogitsProcessor]]]] = {}
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_FQCN:
|
||||
# Scenario: load logitproc based on fully-qualified class name (FQCN)
|
||||
# Inject dummy module which defines logitproc
|
||||
sys.modules[DUMMY_LOGITPROC_MODULE] = dummy_module
|
||||
kwargs["logits_processors"] = [DUMMY_LOGITPROC_FQCN]
|
||||
elif logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_CLASS:
|
||||
# Scenario: load logitproc from provided class object
|
||||
kwargs["logits_processors"] = [DummyLogitsProcessor]
|
||||
|
||||
_run_test(kwargs, logitproc_loaded=True)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("logitproc_source", [
|
||||
CustomLogitprocSource.LOGITPROC_SOURCE_ENTRYPOINT,
|
||||
CustomLogitprocSource.LOGITPROC_SOURCE_FQCN,
|
||||
CustomLogitprocSource.LOGITPROC_SOURCE_CLASS,
|
||||
])
|
||||
def test_pooling_rejects_custom_logitsprocs(
|
||||
monkeypatch, logitproc_source: CustomLogitprocSource):
|
||||
"""Validate that vLLM engine initialization properly rejects custom
|
||||
logitsprocs when the model is a pooling model.
|
||||
|
||||
Use `LLM` entrypoint. We expect `LLM` initialization to fail before the
|
||||
logitproc is actually loaded.
|
||||
|
||||
Scenario 1:
|
||||
* Mock a logitproc entrypoint
|
||||
* Validate that `LLM` does not load the logitproc
|
||||
|
||||
Scenario 2:
|
||||
* Pass custom logitproc to `LLM` constructor
|
||||
* Scenario 2a: via FQCN
|
||||
* Scenario 2b: via class object
|
||||
* Validate that initialization fails with appropriate exception
|
||||
|
||||
Args:
|
||||
monkeypatch: used to set environment variables
|
||||
logitproc_source: what source (entrypoint, fully-qualified class name
|
||||
(FQCN), or class object) the user pulls the
|
||||
logitproc from
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
random.seed(40)
|
||||
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_ENTRYPOINT:
|
||||
# Scenario: vLLM loads a pooling model and ignores a logitproc that is
|
||||
# available at a preconfigured entrypoint
|
||||
|
||||
# Patch in dummy logitproc entrypoint
|
||||
import importlib.metadata
|
||||
importlib.metadata.entry_points = fake_entry_points # type: ignore
|
||||
|
||||
# fork is required for entrypoint patch to be visible to workers,
|
||||
# although they should ignore the entrypoint patch anyway
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "fork")
|
||||
|
||||
llm = LLM(
|
||||
runner="pooling",
|
||||
model=POOLING_MODEL_NAME,
|
||||
gpu_memory_utilization=0.1,
|
||||
)
|
||||
# Require that no logitsprocs have been loaded
|
||||
assert sum([
|
||||
1 for _ in llm.llm_engine.model_executor.driver_worker.worker.
|
||||
model_runner.input_batch.logitsprocs.all
|
||||
]) == 0
|
||||
return
|
||||
|
||||
kwargs: dict[str, list[Union[str, type[LogitsProcessor]]]] = {}
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_FQCN:
|
||||
# Scenario: load logitproc based on fully-qualified class name (FQCN)
|
||||
kwargs["logits_processors"] = [DUMMY_LOGITPROC_FQCN]
|
||||
elif logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_CLASS:
|
||||
# Scenario: load logitproc from provided class object
|
||||
kwargs["logits_processors"] = [DummyLogitsProcessor]
|
||||
|
||||
with pytest.raises(ValueError, match=STR_POOLING_REJECTS_LOGITSPROCS):
|
||||
# Require that loading a pooling model alongside the logitproc raises
|
||||
# the appropriate exception.
|
||||
LLM(
|
||||
runner="pooling",
|
||||
model=POOLING_MODEL_NAME,
|
||||
gpu_memory_utilization=0.1,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import (RemoteOpenAIServerCustom,
|
||||
create_new_process_for_each_test)
|
||||
# yapf: disable
|
||||
from tests.v1.logits_processors.utils import (DUMMY_LOGITPROC_ARG,
|
||||
DUMMY_LOGITPROC_FQCN,
|
||||
DUMMY_LOGITPROC_MODULE,
|
||||
MAX_TOKENS, MODEL_NAME,
|
||||
TEMP_GREEDY, dummy_module)
|
||||
from tests.v1.logits_processors.utils import entry_points as fake_entry_points
|
||||
from tests.v1.logits_processors.utils import prompts
|
||||
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def _server_with_logitproc_entrypoint(
|
||||
env_dict: Optional[dict[str, str]],
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
) -> None:
|
||||
"""Start vLLM server, inject dummy logitproc entrypoint"""
|
||||
|
||||
# Patch `entry_points` to inject logitproc entrypoint
|
||||
import importlib.metadata
|
||||
importlib.metadata.entry_points = fake_entry_points # type: ignore
|
||||
from vllm.entrypoints.cli import main
|
||||
|
||||
# fork is required for workers to see entrypoint patch
|
||||
os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = "fork"
|
||||
if env_dict is not None:
|
||||
os.environ.update(env_dict)
|
||||
|
||||
# Emulate `vllm serve <model> <CLI args>`
|
||||
sys.argv = ["vllm", "serve", model] + vllm_serve_args
|
||||
main.main()
|
||||
|
||||
|
||||
def _server_with_logitproc_module(
|
||||
env_dict: Optional[dict[str, str]],
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
) -> None:
|
||||
"""Start vLLM server, inject module with dummy logitproc"""
|
||||
|
||||
# Patch `modules` to inject dummy logitproc module
|
||||
from vllm.entrypoints.cli import main
|
||||
sys.modules[DUMMY_LOGITPROC_MODULE] = dummy_module
|
||||
|
||||
# fork is required for workers to see entrypoint patch
|
||||
os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = "fork"
|
||||
if env_dict is not None:
|
||||
os.environ.update(env_dict)
|
||||
|
||||
# Emulate `vllm serve <model> <CLI args>`
|
||||
sys.argv = ["vllm", "serve", model] + vllm_serve_args
|
||||
main.main()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="function",
|
||||
params=[[], ["--logits-processors", DUMMY_LOGITPROC_FQCN]])
|
||||
def server(default_server_args, request, monkeypatch):
|
||||
"""Consider two server configurations:
|
||||
(1) --logits-processors cli arg specifies dummy logits processor via fully-
|
||||
qualified class name (FQCN); patch in a dummy logits processor module
|
||||
(2) No --logits-processors cli arg; patch in a dummy logits processor
|
||||
entrypoint
|
||||
"""
|
||||
|
||||
# Test that logitproc info is passed to workers
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
|
||||
|
||||
if request.param:
|
||||
# Launch server, append FQCN argument, inject dummy logitproc module
|
||||
args = default_server_args + request.param
|
||||
_server_fxn = _server_with_logitproc_module
|
||||
else:
|
||||
# Launch server, inject dummy logitproc entrypoint
|
||||
args = default_server_args
|
||||
_server_fxn = _server_with_logitproc_entrypoint
|
||||
|
||||
with RemoteOpenAIServerCustom(MODEL_NAME, args,
|
||||
_server_fxn) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
# General request argument values for these tests
|
||||
api_keyword_args = {
|
||||
# Greedy sampling ensures that requests which receive the `target_token`
|
||||
# arg will decode it in every step
|
||||
"temperature": TEMP_GREEDY,
|
||||
# Since EOS will never be decoded (unless `target_token` is EOS)
|
||||
"max_tokens": MAX_TOKENS,
|
||||
# Return decoded token logprobs (as a way of getting token id)
|
||||
"logprobs": 0,
|
||||
}
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_custom_logitsprocs(client: openai.AsyncOpenAI, model_name: str):
|
||||
"""Test custom logitsprocs when starting OpenAI server from CLI
|
||||
|
||||
Launch vLLM OpenAI-compatible server, configured to load a custom logitproc
|
||||
that has a well-defined behavior (mask out all tokens except one
|
||||
`target_token`).
|
||||
|
||||
Pass in requests, 50% of which pass a `target_token` value
|
||||
in through `extra_body["vllm_xargs"]`, 50% of which do not.
|
||||
|
||||
Validate that requests which activate the custom logitproc, repeat the same
|
||||
token
|
||||
"""
|
||||
|
||||
use_dummy_logitproc = True
|
||||
for prompt in prompts:
|
||||
# Build request arguments
|
||||
request_keyword_args: dict[str, Any] = {
|
||||
**api_keyword_args,
|
||||
}
|
||||
if use_dummy_logitproc:
|
||||
# 50% of requests pass target_token custom arg
|
||||
target_token = random.choice([128, 67])
|
||||
# For requests which activate the dummy logitproc, choose one of
|
||||
# two `target_token` values which are known not to be EOS tokens
|
||||
request_keyword_args["extra_body"] = {
|
||||
"vllm_xargs": {
|
||||
DUMMY_LOGITPROC_ARG: target_token
|
||||
}
|
||||
}
|
||||
batch = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
**request_keyword_args,
|
||||
)
|
||||
|
||||
if use_dummy_logitproc:
|
||||
# Only for requests which activate dummy logitproc - validate that
|
||||
# output token is repeated
|
||||
choices: openai.types.CompletionChoice = batch.choices
|
||||
toks = choices[0].logprobs.tokens
|
||||
if not all([x == toks[0] for x in toks]):
|
||||
raise AssertionError(
|
||||
f"Generated {toks} should all be {toks[0]}")
|
||||
|
||||
# Alternate whether to activate dummy logitproc for each request
|
||||
use_dummy_logitproc = not use_dummy_logitproc
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import types
|
||||
from enum import Enum, auto
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.logits_processor import (LOGITSPROCS_GROUP, BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality)
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
POOLING_MODEL_NAME = "BAAI/bge-base-en-v1.5"
|
||||
DUMMY_LOGITPROC_ARG = "target_token"
|
||||
TEMP_GREEDY = 0.0
|
||||
MAX_TOKENS = 20
|
||||
DUMMY_LOGITPROC_ENTRYPOINT = "dummy_logitproc"
|
||||
DUMMY_LOGITPROC_MODULE = "DummyModule"
|
||||
DUMMY_LOGITPROC_FQCN = f"{DUMMY_LOGITPROC_MODULE}:DummyLogitsProcessor"
|
||||
|
||||
|
||||
class CustomLogitprocSource(Enum):
|
||||
"""How to source a logitproc for testing purposes"""
|
||||
LOGITPROC_SOURCE_NONE = auto() # No custom logitproc
|
||||
LOGITPROC_SOURCE_ENTRYPOINT = auto() # Via entrypoint
|
||||
LOGITPROC_SOURCE_FQCN = auto() # Via fully-qualified class name (FQCN)
|
||||
LOGITPROC_SOURCE_CLASS = auto() # Via provided class object
|
||||
|
||||
|
||||
# Sample prompts.
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
class DummyLogitsProcessor(LogitsProcessor):
|
||||
"""Fake logit processor to support unit testing and examples"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
|
||||
is_pin_memory: bool):
|
||||
self.req_info: dict[int, SamplingParams] = {}
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""Never impacts greedy sampling"""
|
||||
return False
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
if not batch_update:
|
||||
return
|
||||
|
||||
# Process added requests.
|
||||
for index, params, _, _ in batch_update.added:
|
||||
assert params is not None
|
||||
if params.extra_args and (target_token :=
|
||||
params.extra_args.get("target_token")):
|
||||
self.req_info[index] = target_token
|
||||
|
||||
if self.req_info:
|
||||
# Process removed requests.
|
||||
for index in batch_update.removed:
|
||||
self.req_info.pop(index, None)
|
||||
|
||||
# Process moved requests, unidirectional move (a->b) and swap
|
||||
# (a<->b)
|
||||
for adx, bdx, direct in batch_update.moved:
|
||||
a_val = self.req_info.pop(adx, None)
|
||||
b_val = self.req_info.pop(bdx, None)
|
||||
if a_val is not None:
|
||||
self.req_info[bdx] = a_val
|
||||
if direct == MoveDirectionality.SWAP and b_val is not None:
|
||||
self.req_info[adx] = b_val
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if not self.req_info:
|
||||
return logits
|
||||
|
||||
# Save target values before modification
|
||||
rows_list = list(self.req_info.keys())
|
||||
cols = torch.tensor([self.req_info[i] for i in rows_list],
|
||||
dtype=torch.long,
|
||||
device=logits.device)
|
||||
rows = torch.tensor(rows_list, dtype=torch.long, device=logits.device)
|
||||
values_to_keep = logits[rows, cols].clone()
|
||||
|
||||
# Mask all but target tokens
|
||||
logits[rows] = float('-inf')
|
||||
logits[rows, cols] = values_to_keep
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
"""Dummy module with dummy logitproc class"""
|
||||
dummy_module = types.ModuleType(DUMMY_LOGITPROC_MODULE)
|
||||
dummy_module.DummyLogitsProcessor = DummyLogitsProcessor # type: ignore
|
||||
|
||||
|
||||
class EntryPoint:
|
||||
"""Dummy entrypoint class for logitsprocs testing"""
|
||||
|
||||
def __init__(self):
|
||||
self.name = DUMMY_LOGITPROC_ENTRYPOINT
|
||||
self.value = DUMMY_LOGITPROC_FQCN
|
||||
|
||||
def load(self):
|
||||
return DummyLogitsProcessor
|
||||
|
||||
|
||||
class EntryPoints(list):
|
||||
"""Dummy EntryPoints class for logitsprocs testing"""
|
||||
|
||||
def __init__(self, group: str):
|
||||
# Emulate list-like functionality
|
||||
eps = [EntryPoint()] if group == LOGITSPROCS_GROUP else []
|
||||
super().__init__(eps)
|
||||
# Extra attributes
|
||||
self.names = [ep.name for ep in eps]
|
||||
|
||||
|
||||
"""Fake version of importlib.metadata.entry_points"""
|
||||
entry_points = lambda group: EntryPoints(group)
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessorManager
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.rejection_sampler import (PLACEHOLDER_TOKEN_ID,
|
||||
RejectionSampler)
|
||||
@@ -69,7 +69,7 @@ def create_sampling_metadata(
|
||||
output_token_ids=[],
|
||||
allowed_token_ids_mask=None,
|
||||
bad_words_token_ids={},
|
||||
logitsprocs=LogitsProcessorManager(),
|
||||
logitsprocs=LogitsProcessors(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessorManager
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
@@ -173,7 +173,7 @@ def _create_default_sampling_metadata(
|
||||
no_penalties=True,
|
||||
allowed_token_ids_mask=None,
|
||||
bad_words_token_ids={},
|
||||
logitsprocs=LogitsProcessorManager(),
|
||||
logitsprocs=LogitsProcessors(),
|
||||
)
|
||||
return fake_sampling_metadata
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessorManager
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.worker.block_table import BlockTable, MultiGroupBlockTable
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
@@ -169,7 +169,7 @@ def _construct_expected_sampling_metadata(
|
||||
and all(x == 1 for x in repetition_penalties)),
|
||||
allowed_token_ids_mask=allowed_token_ids_mask,
|
||||
bad_words_token_ids=bad_words_token_ids,
|
||||
logitsprocs=LogitsProcessorManager(),
|
||||
logitsprocs=LogitsProcessors(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user