[CI/CD] add neuron docker and ci test scripts (#3571)

This commit is contained in:
Liangfu Chen
2024-04-18 15:26:01 -07:00
committed by GitHub
parent 87fa80c91f
commit cd2f63fb36
6 changed files with 103 additions and 4 deletions
+21 -1
View File
@@ -1,9 +1,10 @@
from typing import Dict, List, Set, Tuple
from vllm.executor.executor_base import ExecutorBase
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
from vllm.utils import make_async
logger = init_logger(__name__)
@@ -73,3 +74,22 @@ class NeuronExecutor(ExecutorBase):
# NeuronExecutor will always be healthy as long as
# it's running.
return
class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
async def execute_model_async(
self,
seq_group_metadata_list: List[SequenceGroupMetadata],
blocks_to_swap_in: Dict[int, int],
blocks_to_swap_out: Dict[int, int],
blocks_to_copy: Dict[int, List[int]],
) -> SamplerOutput:
output = await make_async(self.driver_worker.execute_model)(
seq_group_metadata_list=seq_group_metadata_list, )
return output
async def check_health_async(self) -> None:
# NeuronExecutor will always be healthy as long as
# it's running.
return