[CI] GSM8K eval integration test for KV offloading (#46893)

Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-07-08 17:59:48 -04:00
committed by GitHub
co-authored by Claude Opus 4.6 Itay Etelis Itay Etelis Or Ozeri mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 0d2f4e7c9c
commit dcdd756d75
5 changed files with 330 additions and 10 deletions
+6 -5
View File
@@ -3,7 +3,7 @@
from fastapi import APIRouter, FastAPI, Query, Request
from fastapi.responses import Response
from fastapi.responses import JSONResponse, Response
from vllm.engine.protocol import EngineClient
from vllm.logger import init_logger
@@ -29,18 +29,19 @@ async def reset_prefix_cache(
Optionally, if the query parameter `reset_external=true`
also resets the external (connector-managed) prefix cache.
Note that we currently do not check if the prefix cache
is successfully reset in the API server.
Returns `{"success": bool}`. The reset fails (`success=false`) while
blocks are still held, e.g. by running requests or in-flight async KV
offload transfers; callers may retry.
Example:
POST /reset_prefix_cache?reset_external=true
"""
logger.info("Resetting prefix cache...")
await engine_client(raw_request).reset_prefix_cache(
success = await engine_client(raw_request).reset_prefix_cache(
reset_running_requests, reset_external
)
return Response(status_code=200)
return JSONResponse(content={"success": bool(success)})
@router.post("/reset_mm_cache")