Tyler Michael SmithClaude Opus 4.6Itay EtelisItay EtelisOr Ozerimergify[bot] <37929162+mergify[bot]@users.noreply.github.com>Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
71 lines
2.1 KiB
Python
71 lines
2.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
|
|
from fastapi import APIRouter, FastAPI, Query, Request
|
|
from fastapi.responses import JSONResponse, Response
|
|
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.logger import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def engine_client(request: Request) -> EngineClient:
|
|
return request.app.state.engine_client
|
|
|
|
|
|
@router.post("/reset_prefix_cache")
|
|
async def reset_prefix_cache(
|
|
raw_request: Request,
|
|
reset_running_requests: bool = Query(default=False),
|
|
reset_external: bool = Query(default=False),
|
|
):
|
|
"""
|
|
Reset the local prefix cache.
|
|
|
|
Optionally, if the query parameter `reset_external=true`
|
|
also resets the external (connector-managed) prefix cache.
|
|
|
|
Returns `{"success": bool}`. The reset fails (`success=false`) while
|
|
blocks are still held, e.g. by running requests or in-flight async KV
|
|
offload transfers; callers may retry.
|
|
|
|
Example:
|
|
POST /reset_prefix_cache?reset_external=true
|
|
"""
|
|
logger.info("Resetting prefix cache...")
|
|
|
|
success = await engine_client(raw_request).reset_prefix_cache(
|
|
reset_running_requests, reset_external
|
|
)
|
|
return JSONResponse(content={"success": bool(success)})
|
|
|
|
|
|
@router.post("/reset_mm_cache")
|
|
async def reset_mm_cache(raw_request: Request):
|
|
"""
|
|
Reset the multi-modal cache. Note that we currently do not check if the
|
|
multi-modal cache is successfully reset in the API server.
|
|
"""
|
|
logger.info("Resetting multi-modal cache...")
|
|
await engine_client(raw_request).reset_mm_cache()
|
|
return Response(status_code=200)
|
|
|
|
|
|
@router.post("/reset_encoder_cache")
|
|
async def reset_encoder_cache(raw_request: Request):
|
|
"""
|
|
Reset the encoder cache. Note that we currently do not check if the
|
|
encoder cache is successfully reset in the API server.
|
|
"""
|
|
logger.info("Resetting encoder cache...")
|
|
await engine_client(raw_request).reset_encoder_cache()
|
|
return Response(status_code=200)
|
|
|
|
|
|
def attach_router(app: FastAPI):
|
|
app.include_router(router)
|