[Core] Eliminate parallel worker per-step task scheduling overhead (#4894)

This commit is contained in:
Nick Hill
2024-05-23 06:17:27 +09:00
committed by GitHub
parent 97b030005c
commit eb6d3c264d
12 changed files with 350 additions and 211 deletions
+4 -3
View File
@@ -1,7 +1,7 @@
import importlib
import os
from abc import ABC, abstractmethod
from typing import Dict, List, Set, Tuple
from typing import Dict, List, Optional, Set, Tuple
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -48,8 +48,9 @@ class WorkerBase(ABC):
@abstractmethod
def execute_model(
self,
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
self,
execute_model_req: Optional[ExecuteModelRequest] = None
) -> List[SamplerOutput]:
"""Executes at least one model step on the given sequences, unless no
sequences are provided."""
raise NotImplementedError