[Bugfix][Core] Fix get decoding config from ray (#4335)

This commit is contained in:
Roy
2024-04-27 11:30:08 +00:00
committed by GitHub
parent 3da24c2df7
commit 7134303cbb
6 changed files with 174 additions and 3 deletions
+9 -1
View File
@@ -7,7 +7,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
from transformers import PreTrainedTokenizer
from vllm.config import ModelConfig
from vllm.config import DecodingConfig, ModelConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.llm_engine import LLMEngine
from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -697,6 +697,14 @@ class AsyncLLMEngine:
else:
return self.engine.get_model_config()
async def get_decoding_config(self) -> DecodingConfig:
"""Get the decoding configuration of the vLLM engine."""
if self.engine_use_ray:
return await self.engine.get_decoding_config.remote( # type: ignore
)
else:
return self.engine.get_decoding_config()
async def do_log_stats(self) -> None:
if self.engine_use_ray:
await self.engine.do_log_stats.remote() # type: ignore
+4
View File
@@ -467,6 +467,10 @@ class LLMEngine:
"""Gets the model configuration."""
return self.model_config
def get_decoding_config(self) -> DecodingConfig:
"""Gets the decoding configuration."""
return self.decoding_config
def get_num_unfinished_requests(self) -> int:
"""Gets the number of unfinished requests."""
return self.scheduler.get_num_unfinished_seq_groups()