[Frontend] Support strict mode for tool calling (#45003)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-06-12 07:51:48 +00:00
committed by GitHub
co-authored by cjackal mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bd59c913bc
commit 2043258dec
29 changed files with 692 additions and 1956 deletions
+6 -7
View File
@@ -200,6 +200,7 @@ if TYPE_CHECKING:
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
@@ -227,7 +228,6 @@ if TYPE_CHECKING:
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
VLLM_SYSTEM_START_DATE: str | None = None
VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = False
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
@@ -1536,6 +1536,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
),
# Enforce function parameter schemas in structural-tag based tool calling.
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: os.getenv(
"VLLM_ENFORCE_STRICT_TOOL_CALLING", "True"
).lower()
in ("true", "1"),
# Control the max chunk bytes (in MB) for the rpc message queue.
# Object larger than this threshold will be broadcast to worker
# processes via zmq.
@@ -1659,12 +1664,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
),
# When 1,the model structural tags will be used to enforce the model
# output conforming to the model's tool-calling format and schema.
# Default 0 (off).
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: bool(
int(os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "0"))
),
# Add optional custom scopes for profiling, disable to avoid overheads
"VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))