[Bugfix][Frontend] Lazy-import model_hosting_container_standards to prevent log suppression (#51366)

Signed-off-by: wenjinhust <wenjin.sh@huawei.com>
This commit is contained in:
wenjinhust
2026-09-16 08:25:49 +00:00
committed by GitHub
parent 8be5205abb
commit 0d8173d153
2 changed files with 87 additions and 4 deletions
+28 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import model_hosting_container_standards.sagemaker as sagemaker_standards
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response
@@ -22,6 +21,12 @@ router = APIRouter()
def attach_router(app: FastAPI):
"""Attach the LoRA adapter load/unload endpoints to the API server.
Does nothing when dynamic LoRA updating is disabled. Handler levels are
snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
return
@@ -30,6 +35,23 @@ def attach_router(app: FastAPI):
"This should ONLY be used for local development!"
)
snapshot = [
(h, h.level)
for lg in (logging.getLogger(), logging.getLogger("vllm"))
for h in lg.handlers
]
try:
_attach_router(app)
finally:
for handler, level in snapshot:
handler.setLevel(level)
def _attach_router(app: FastAPI):
"""Register the LoRA adapter load/unload routes on the app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
@sagemaker_standards.register_load_adapter_handler(
request_shape={
"lora_name": "body.name",
@@ -40,6 +62,8 @@ def attach_router(app: FastAPI):
)
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.load_lora_adapter(request)
if isinstance(response, ErrorResponse):
@@ -60,6 +84,8 @@ def attach_router(app: FastAPI):
async def unload_lora_adapter(
request: UnloadLoRAAdapterRequest, raw_request: Request
):
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
the serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.unload_lora_adapter(request)
if isinstance(response, ErrorResponse):
+59 -2
View File
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from typing import Any
import model_hosting_container_standards.sagemaker as sagemaker_standards
import pydantic
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
@@ -27,11 +27,55 @@ GetHandlerFn = Callable[[Request], BaseServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
"""Snapshot handler levels for loggers that may be affected by third-party
logging configuration side effects (e.g. model_hosting_container_standards
calling configure_root_logger() at import time)."""
loggers = [logging.getLogger(), logging.getLogger("vllm")]
seen: set[int] = set()
snapshot: list[tuple[logging.Handler, int]] = []
for logger in loggers:
for handler in logger.handlers:
if id(handler) not in seen:
seen.add(id(handler))
snapshot.append((handler, handler.level))
return snapshot
def _restore_handler_levels(
snapshot: list[tuple[logging.Handler, int]],
) -> None:
"""Restore handler levels from a snapshot."""
for handler, level in snapshot:
handler.setLevel(level)
def attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Attach the SageMaker hosting endpoints to the API server.
Handler levels are snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
snapshot = _snapshot_handler_levels()
try:
_attach_router(app, supported_tasks, model_config)
finally:
_restore_handler_levels(snapshot)
def _attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Register the SageMaker hosting routes (/ping, /invocations) on the
app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
@@ -99,4 +143,17 @@ def attach_router(
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
return sagemaker_standards.bootstrap(app)
"""Bootstrap the app with the SageMaker hosting standards.
Handler levels are restored right after the import because importing
model_hosting_container_standards may reconfigure root logging, and
bootstrap must run with the original levels.
"""
snapshot = _snapshot_handler_levels()
try:
import model_hosting_container_standards.sagemaker as sagemaker_standards
app = sagemaker_standards.bootstrap(app)
finally:
_restore_handler_levels(snapshot)
return app