[Feat] Support thinking_token_budget in Model Runner V2 (#46727)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -72,9 +72,7 @@ def server():
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
# thinking_token_budget is not yet supported by the V2 model runner.
|
||||
env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -90,9 +88,7 @@ def server_with_auto_reasoning_config():
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
# thinking_token_budget is not yet supported by the V2 model runner.
|
||||
env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -126,8 +122,7 @@ def server_qwen35_fp8_mtp_tp2():
|
||||
}
|
||||
),
|
||||
]
|
||||
# thinking_token_budget is not yet supported by the V2 model runner.
|
||||
env_dict: dict[str, str] = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
|
||||
env_dict: dict[str, str] = {}
|
||||
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
|
||||
# on 0,1 do not exhaust memory on the same devices as this worker.
|
||||
if current_platform.device_count() >= 4:
|
||||
|
||||
Reference in New Issue
Block a user