Fix performance when --generation-config is not None (#14223)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-03-04 20:59:22 +01:00
committed by GitHub
parent beebf4742a
commit 9badee53de
4 changed files with 23 additions and 25 deletions
+6 -4
View File
@@ -244,6 +244,7 @@ class LLM:
engine_args, usage_context=UsageContext.LLM_CLASS)
self.request_counter = Counter()
self.default_sampling_params: Union[dict[str, Any], None] = None
@staticmethod
def get_engine_class() -> type[LLMEngine]:
@@ -268,10 +269,11 @@ class LLM:
tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
def get_default_sampling_params(self) -> SamplingParams:
diff_sampling_param = (
self.llm_engine.model_config.get_diff_sampling_param())
if diff_sampling_param:
return SamplingParams.from_optional(**diff_sampling_param)
if self.default_sampling_params is None:
self.default_sampling_params = (
self.llm_engine.model_config.get_diff_sampling_param())
if self.default_sampling_params:
return SamplingParams.from_optional(**self.default_sampling_params)
return SamplingParams()
@overload