[Bugfix] Fix SHM cache initialization (#26427)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-10-09 02:48:04 -07:00
committed by GitHub
parent dc7976dd9f
commit 4bdf7ac593
30 changed files with 357 additions and 417 deletions
+3 -12
View File
@@ -14,7 +14,6 @@ import torch
from prometheus_client import start_http_server
from tqdm import tqdm
from vllm.config import VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
@@ -328,7 +327,6 @@ async def run_request(
async def run_batch(
engine_client: EngineClient,
vllm_config: VllmConfig,
args: Namespace,
) -> None:
if args.served_model_name is not None:
@@ -345,22 +343,19 @@ async def run_batch(
BaseModelPath(name=name, model_path=args.model) for name in served_model_names
]
model_config = vllm_config.model_config
model_config = engine_client.model_config
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported_tasks: %s", supported_tasks)
logger.info("Supported tasks: %s", supported_tasks)
# Create the openai serving objects.
openai_serving_models = OpenAIServingModels(
engine_client=engine_client,
model_config=model_config,
base_model_paths=base_model_paths,
lora_modules=None,
)
openai_serving_chat = (
OpenAIServingChat(
engine_client,
model_config,
openai_serving_models,
args.response_role,
request_logger=request_logger,
@@ -374,7 +369,6 @@ async def run_batch(
openai_serving_embedding = (
OpenAIServingEmbedding(
engine_client,
model_config,
openai_serving_models,
request_logger=request_logger,
chat_template=None,
@@ -392,7 +386,6 @@ async def run_batch(
openai_serving_scores = (
ServingScores(
engine_client,
model_config,
openai_serving_models,
request_logger=request_logger,
)
@@ -509,9 +502,7 @@ async def main(args: Namespace):
usage_context=UsageContext.OPENAI_BATCH_RUNNER,
disable_frontend_multiprocessing=False,
) as engine_client:
vllm_config = await engine_client.get_vllm_config()
await run_batch(engine_client, vllm_config, args)
await run_batch(engine_client, args)
if __name__ == "__main__":