[Chore] Cleanup guided namespace, move to structured outputs config (#22772)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Aaron Pham
2025-09-18 09:20:27 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 05b044e698
commit 29283e8976
51 changed files with 579 additions and 806 deletions
+1 -1
View File
@@ -184,7 +184,7 @@ def sample_enum_json_schema():
@pytest.fixture
def sample_guided_choice():
def sample_structured_outputs_choices():
return [
"Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
"Ruby", "Swift", "Kotlin"
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from contextlib import nullcontext
from vllm_test_utils import BlameResult, blame
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.sampling_params import GuidedDecodingParams
def run_normal():
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# Create an LLM without guided decoding as a baseline.
llm = LLM(model="distilbert/distilgpt2",
enforce_eager=True,
gpu_memory_utilization=0.3)
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
# Destroy the LLM object and free up the GPU memory.
del llm
cleanup_dist_env_and_memory()
def run_xgrammar(sample_regex):
# Create an LLM with guided decoding enabled.
llm = LLM(model="distilbert/distilgpt2",
enforce_eager=True,
guided_decoding_backend="xgrammar",
gpu_memory_utilization=0.3)
prompt = f"Give an example IPv4 address with this regex: {sample_regex}"
guided_decoding = GuidedDecodingParams(regex=sample_regex)
sampling_params = SamplingParams(temperature=0.8,
top_p=0.95,
guided_decoding=guided_decoding)
outputs = llm.generate(
prompts=[prompt] * 2,
sampling_params=sampling_params,
use_tqdm=True,
)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
def test_lazy_outlines(sample_regex):
"""If users don't use guided decoding, outlines should not be imported.
"""
# make sure outlines is not imported
module_name = "outlines"
# In CI, we only check finally if the module is imported.
# If it is indeed imported, we can rerun the test with `use_blame=True`,
# which will trace every function call to find the first import location,
# and help find the root cause.
# We don't run it in CI by default because it is slow.
use_blame = False
context = blame(
lambda: module_name in sys.modules) if use_blame else nullcontext()
with context as result:
run_normal()
run_xgrammar(sample_regex)
if use_blame:
assert isinstance(result, BlameResult)
print(f"the first import location is:\n{result.trace_stack}")
assert module_name not in sys.modules, (
f"Module {module_name} is imported. To see the first"
f" import location, run the test with `use_blame=True`.")
+59 -64
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# imports for guided decoding tests
# imports for structured outputs tests
import json
from typing import Optional
@@ -480,10 +480,11 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
@pytest.mark.asyncio
async def test_guided_choice_chat(client: openai.AsyncOpenAI,
sample_guided_choice, is_v1_server: bool):
async def test_structured_outputs_choice_chat(
client: openai.AsyncOpenAI, sample_structured_outputs_choices,
is_v1_server: bool):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("Structured outputs is only supported in v1 engine")
messages = [{
"role": "system",
"content": "you are a helpful assistant"
@@ -498,9 +499,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
messages=messages,
max_completion_tokens=10,
temperature=0.7,
extra_body=dict(guided_choice=sample_guided_choice))
extra_body=dict(
structured_outputs={"choice": sample_structured_outputs_choices}))
choice1 = chat_completion.choices[0].message.content
assert choice1 in sample_guided_choice
assert choice1 in sample_structured_outputs_choices
messages.append({"role": "assistant", "content": choice1})
messages.append({
@@ -512,17 +514,19 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
messages=messages,
max_completion_tokens=10,
temperature=0.7,
extra_body=dict(guided_choice=sample_guided_choice))
extra_body=dict(
structured_outputs={"choice": sample_structured_outputs_choices}))
choice2 = chat_completion.choices[0].message.content
assert choice2 in sample_guided_choice
assert choice2 in sample_structured_outputs_choices
assert choice1 != choice2
@pytest.mark.asyncio
async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
is_v1_server: bool):
async def test_structured_outputs_json_chat(client: openai.AsyncOpenAI,
sample_json_schema,
is_v1_server: bool):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("Structured outputs is only supported in v1 engine")
messages = [{
"role": "system",
@@ -538,7 +542,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
model=MODEL_NAME,
messages=messages,
max_completion_tokens=1000,
extra_body=dict(guided_json=sample_json_schema))
extra_body=dict(structured_outputs={"json": sample_json_schema}))
message = chat_completion.choices[0].message
assert message.content is not None
json1 = json.loads(message.content)
@@ -555,7 +559,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
model=MODEL_NAME,
messages=messages,
max_completion_tokens=1000,
extra_body=dict(guided_json=sample_json_schema))
extra_body=dict(structured_outputs={"json": sample_json_schema}))
message = chat_completion.choices[0].message
assert message.content is not None
json2 = json.loads(message.content)
@@ -565,10 +569,10 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
@pytest.mark.asyncio
async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
is_v1_server: bool):
async def test_structured_outputs_regex_chat(client: openai.AsyncOpenAI,
sample_regex, is_v1_server: bool):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("Structured outputs is only supported in v1 engine")
messages = [{
"role": "system",
@@ -583,7 +587,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
model=MODEL_NAME,
messages=messages,
max_completion_tokens=20,
extra_body=dict(guided_regex=sample_regex))
extra_body=dict(structured_outputs={"regex": sample_regex}))
ip1 = chat_completion.choices[0].message.content
assert ip1 is not None
assert re.fullmatch(sample_regex, ip1) is not None
@@ -594,7 +598,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
model=MODEL_NAME,
messages=messages,
max_completion_tokens=20,
extra_body=dict(guided_regex=sample_regex))
extra_body=dict(structured_outputs={"regex": sample_regex}))
ip2 = chat_completion.choices[0].message.content
assert ip2 is not None
assert re.fullmatch(sample_regex, ip2) is not None
@@ -602,7 +606,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
@pytest.mark.asyncio
async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
async def test_structured_outputs_type_error(client: openai.AsyncOpenAI):
messages = [{
"role": "system",
"content": "you are a helpful assistant"
@@ -614,17 +618,19 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
}]
with pytest.raises(openai.BadRequestError):
_ = await client.chat.completions.create(model=MODEL_NAME,
messages=messages,
extra_body=dict(guided_regex={
1: "Python",
2: "C++"
}))
_ = await client.chat.completions.create(
model=MODEL_NAME,
messages=messages,
extra_body=dict(
structured_outputs={"regex": {
1: "Python",
2: "C++"
}}))
@pytest.mark.asyncio
async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
sample_guided_choice):
async def test_structured_outputs_choice_chat_logprobs(
client: openai.AsyncOpenAI, sample_structured_outputs_choices):
messages = [{
"role": "system",
@@ -641,7 +647,8 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
max_completion_tokens=10,
logprobs=True,
top_logprobs=5,
extra_body=dict(guided_choice=sample_guided_choice))
extra_body=dict(
structured_outputs={"choice": sample_structured_outputs_choices}))
assert chat_completion.choices[0].logprobs is not None
assert chat_completion.choices[0].logprobs.content is not None
@@ -663,10 +670,23 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
}, {
"role":
"user",
"content":
f"Give an example JSON for an employee profile that "
f"fits this schema: {sample_json_schema}"
"content": ("Give an example JSON for an employee "
"profile using the specified tool.")
}]
tools = [{
"type": "function",
"function": {
"name": "dummy_function_name",
"description": "This is a dummy function",
"parameters": sample_json_schema
}
}]
tool_choice = {
"type": "function",
"function": {
"name": "dummy_function_name"
}
}
# non-streaming
@@ -674,20 +694,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
model=MODEL_NAME,
messages=messages,
max_completion_tokens=1000,
tools=[{
"type": "function",
"function": {
"name": "dummy_function_name",
"description": "This is a dummy function",
"parameters": sample_json_schema
}
}],
tool_choice={
"type": "function",
"function": {
"name": "dummy_function_name"
}
},
tools=tools,
tool_choice=tool_choice,
)
message = chat_completion.choices[0].message
assert len(message.content) == 0
@@ -705,25 +713,12 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
# streaming
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=messages,
max_completion_tokens=1000,
tools=[{
"type": "function",
"function": {
"name": "dummy_function_name",
"description": "This is a dummy function",
"parameters": sample_json_schema
}
}],
tool_choice={
"type": "function",
"function": {
"name": "dummy_function_name"
}
},
stream=True)
stream = await client.chat.completions.create(model=MODEL_NAME,
messages=messages,
max_completion_tokens=1000,
tools=tools,
tool_choice=tool_choice,
stream=True)
output = []
finish_reason_count = 0
+39 -40
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# imports for guided decoding tests
# imports for structured outputs tests
import json
import os
from typing import Optional
@@ -23,8 +23,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
# technically these adapters use a different base model,
# but we're not testing generation quality here
GUIDED_DECODING_BACKENDS = ["outlines", "xgrammar", "guidance"]
@pytest.fixture(scope="module")
def default_server_args(zephyr_lora_files):
@@ -595,12 +593,13 @@ async def test_allowed_token_ids(client: openai.AsyncOpenAI):
@pytest.mark.asyncio
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
async def test_guided_json_completion(client: openai.AsyncOpenAI,
guided_decoding_backend: str,
sample_json_schema, is_v1_server: bool):
async def test_structured_outputs_json_completion(
client: openai.AsyncOpenAI,
sample_json_schema,
is_v1_server: bool,
):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("structured outputs is only supported in v1 engine")
completion = await client.completions.create(
model=MODEL_NAME,
@@ -609,8 +608,7 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
n=3,
temperature=1.0,
max_tokens=500,
extra_body=dict(guided_json=sample_json_schema,
guided_decoding_backend=guided_decoding_backend))
extra_body=dict(structured_outputs=dict(json=sample_json_schema)))
assert completion.id is not None
assert len(completion.choices) == 3
@@ -620,12 +618,13 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
@pytest.mark.asyncio
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
async def test_guided_regex_completion(client: openai.AsyncOpenAI,
guided_decoding_backend: str,
sample_regex, is_v1_server: bool):
async def test_structured_outputs_regex_completion(
client: openai.AsyncOpenAI,
sample_regex,
is_v1_server: bool,
):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("structured outputs is only supported in v1 engine")
completion = await client.completions.create(
model=MODEL_NAME,
@@ -633,8 +632,7 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
n=3,
temperature=1.0,
max_tokens=20,
extra_body=dict(guided_regex=sample_regex,
guided_decoding_backend=guided_decoding_backend))
extra_body=dict(structured_outputs=dict(regex=sample_regex)))
assert completion.id is not None
assert len(completion.choices) == 3
@@ -644,13 +642,13 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
@pytest.mark.asyncio
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
async def test_guided_choice_completion(client: openai.AsyncOpenAI,
guided_decoding_backend: str,
sample_guided_choice,
is_v1_server: bool):
async def test_structured_outputs_choice_completion(
client: openai.AsyncOpenAI,
sample_structured_outputs_choices,
is_v1_server: bool,
):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("structured outputs is only supported in v1 engine")
completion = await client.completions.create(
model=MODEL_NAME,
@@ -658,20 +656,21 @@ async def test_guided_choice_completion(client: openai.AsyncOpenAI,
n=2,
temperature=1.0,
max_tokens=10,
extra_body=dict(guided_choice=sample_guided_choice,
guided_decoding_backend=guided_decoding_backend))
extra_body=dict(structured_outputs=dict(
choice=sample_structured_outputs_choices)))
assert completion.id is not None
assert len(completion.choices) == 2
for i in range(2):
assert completion.choices[i].text in sample_guided_choice
assert completion.choices[i].text in sample_structured_outputs_choices
@pytest.mark.asyncio
async def test_guided_grammar(client: openai.AsyncOpenAI,
sample_sql_statements, is_v1_server: bool):
async def test_structured_outputs_grammar(client: openai.AsyncOpenAI,
sample_sql_statements,
is_v1_server: bool):
if not is_v1_server:
pytest.skip("Guided grammar is only supported in v1 engine")
pytest.skip("grammar is only supported in v1 engine")
completion = await client.completions.create(
model=MODEL_NAME,
@@ -679,7 +678,8 @@ async def test_guided_grammar(client: openai.AsyncOpenAI,
"table_1 where it is equals to 1"),
temperature=1.0,
max_tokens=500,
extra_body=dict(guided_grammar=sample_sql_statements))
extra_body=dict(
structured_outputs=dict(grammar=sample_sql_statements), ))
content = completion.choices[0].text
@@ -730,27 +730,26 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
@pytest.mark.asyncio
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
guided_decoding_backend: str,
sample_json_schema, sample_regex,
is_v1_server: bool):
async def test_structured_outputs_type_error(client: openai.AsyncOpenAI,
sample_json_schema, sample_regex,
is_v1_server: bool):
if not is_v1_server:
pytest.skip("Guided decoding is only supported in v1 engine")
pytest.skip("structured outputs is only supported in v1 engine")
with pytest.raises(openai.BadRequestError):
_ = await client.completions.create(
model=MODEL_NAME,
prompt="Give an example JSON that fits this schema: 42",
extra_body=dict(guided_json=42,
guided_decoding_backend=guided_decoding_backend))
extra_body=dict(structured_outputs=dict(json=42)))
with pytest.raises(openai.BadRequestError):
_ = await client.completions.create(
model=MODEL_NAME,
prompt="Give an example string that fits this regex",
extra_body=dict(guided_regex=sample_regex,
guided_json=sample_json_schema))
extra_body=dict(structured_outputs=dict(
regex=sample_regex,
json=sample_json_schema,
)))
@pytest.mark.asyncio
@@ -142,7 +142,7 @@ def server(): # noqa: F811
"--dtype",
"half",
"--enable-auto-tool-choice",
"--guided-decoding-backend",
"--structured-outputs-config.backend",
"xgrammar",
"--tool-call-parser",
"hermes",
@@ -225,7 +225,7 @@ def k2_server(): # noqa: F811
"--dtype",
"half",
"--enable-auto-tool-choice",
"--guided-decoding-backend",
"--structured-outputs-config.backend",
"xgrammar",
"--tool-call-parser",
"hermes",
@@ -102,12 +102,14 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
if "custom" in tool_call:
return False
# Sometimes guided_grammar is generated to be empty
# Sometimes structured_outputs.grammar is generated to be empty
# Causing a server error in EBNF grammar parsing
# https://github.com/vllm-project/vllm/pull/22587#issuecomment-3195253421
guided_grammar = case.body.get("guided_grammar")
structured_outputs = case.body.get("structured_outputs", {})
grammar = structured_outputs.get("grammar") if isinstance(
structured_outputs, dict) else None
if guided_grammar == '':
if grammar == '':
# Allow None (will be handled as no grammar)
# But skip empty strings
return False
@@ -3,7 +3,7 @@
import io
# imports for guided decoding tests
# imports for structured outputs tests
import openai
import pybase64
import pytest
@@ -333,7 +333,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
"role": "user",
"content": "what is 1+1?"
}],
guided_decoding_backend="outlines",
)
with suppress(Exception):
@@ -378,7 +377,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
"role": "user",
"content": "what is 1+1?"
}],
guided_decoding_backend="outlines",
)
with suppress(Exception):
@@ -433,7 +431,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
"role": "user",
"content": "what is 1+1?"
}],
guided_decoding_backend="outlines",
)
with suppress(Exception):
@@ -489,7 +486,6 @@ async def test_serving_chat_could_load_correct_generation_config():
"role": "user",
"content": "what is 1+1?"
}],
guided_decoding_backend="outlines",
)
with suppress(Exception):
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# imports for guided decoding tests
# imports for structured outputs tests
import io
import json
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
# imports for guided decoding tests
# imports for structured outputs tests
import json
import httpx