[Feat] Support thinking_token_budget in Model Runner V2 (#46727)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Chauncey
2026-08-07 03:36:18 +00:00
committed by GitHub
co-authored by Nick Hill
parent 5ac2684976
commit 72c0d67657
10 changed files with 1122 additions and 43 deletions
@@ -72,9 +72,7 @@ def server():
"0.4",
"--no-async-scheduling",
]
# thinking_token_budget is not yet supported by the V2 model runner.
env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -90,9 +88,7 @@ def server_with_auto_reasoning_config():
"0.4",
"--no-async-scheduling",
]
# thinking_token_budget is not yet supported by the V2 model runner.
env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -126,8 +122,7 @@ def server_qwen35_fp8_mtp_tp2():
}
),
]
# thinking_token_budget is not yet supported by the V2 model runner.
env_dict: dict[str, str] = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
env_dict: dict[str, str] = {}
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
# on 0,1 do not exhaust memory on the same devices as this worker.
if current_platform.device_count() >= 4: