[Frontend] Support strict mode for tool calling (#45003)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
cjackal
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bd59c913bc
commit
2043258dec
+6
-7
@@ -200,6 +200,7 @@ if TYPE_CHECKING:
|
||||
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
|
||||
@@ -227,7 +228,6 @@ if TYPE_CHECKING:
|
||||
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
|
||||
VLLM_SYSTEM_START_DATE: str | None = None
|
||||
VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = False
|
||||
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
|
||||
VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
|
||||
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
|
||||
@@ -1536,6 +1536,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
|
||||
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
|
||||
),
|
||||
# Enforce function parameter schemas in structural-tag based tool calling.
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: os.getenv(
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING", "True"
|
||||
).lower()
|
||||
in ("true", "1"),
|
||||
# Control the max chunk bytes (in MB) for the rpc message queue.
|
||||
# Object larger than this threshold will be broadcast to worker
|
||||
# processes via zmq.
|
||||
@@ -1659,12 +1664,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
|
||||
int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
|
||||
),
|
||||
# When 1,the model structural tags will be used to enforce the model
|
||||
# output conforming to the model's tool-calling format and schema.
|
||||
# Default 0 (off).
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: bool(
|
||||
int(os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "0"))
|
||||
),
|
||||
# Add optional custom scopes for profiling, disable to avoid overheads
|
||||
"VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
|
||||
int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))
|
||||
|
||||
Reference in New Issue
Block a user