[CI] GSM8K eval integration test for KV offloading (#46893)
Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
Itay Etelis
Itay Etelis
Or Ozeri
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0d2f4e7c9c
commit
dcdd756d75
+6
-5
@@ -3,7 +3,7 @@
|
||||
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Query, Request
|
||||
from fastapi.responses import Response
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
@@ -29,18 +29,19 @@ async def reset_prefix_cache(
|
||||
Optionally, if the query parameter `reset_external=true`
|
||||
also resets the external (connector-managed) prefix cache.
|
||||
|
||||
Note that we currently do not check if the prefix cache
|
||||
is successfully reset in the API server.
|
||||
Returns `{"success": bool}`. The reset fails (`success=false`) while
|
||||
blocks are still held, e.g. by running requests or in-flight async KV
|
||||
offload transfers; callers may retry.
|
||||
|
||||
Example:
|
||||
POST /reset_prefix_cache?reset_external=true
|
||||
"""
|
||||
logger.info("Resetting prefix cache...")
|
||||
|
||||
await engine_client(raw_request).reset_prefix_cache(
|
||||
success = await engine_client(raw_request).reset_prefix_cache(
|
||||
reset_running_requests, reset_external
|
||||
)
|
||||
return Response(status_code=200)
|
||||
return JSONResponse(content={"success": bool(success)})
|
||||
|
||||
|
||||
@router.post("/reset_mm_cache")
|
||||
|
||||
Reference in New Issue
Block a user