Fix performance when --generation-config is not None (#14223)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -244,6 +244,7 @@ class LLM:
|
||||
engine_args, usage_context=UsageContext.LLM_CLASS)
|
||||
|
||||
self.request_counter = Counter()
|
||||
self.default_sampling_params: Union[dict[str, Any], None] = None
|
||||
|
||||
@staticmethod
|
||||
def get_engine_class() -> type[LLMEngine]:
|
||||
@@ -268,10 +269,11 @@ class LLM:
|
||||
tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
|
||||
|
||||
def get_default_sampling_params(self) -> SamplingParams:
|
||||
diff_sampling_param = (
|
||||
self.llm_engine.model_config.get_diff_sampling_param())
|
||||
if diff_sampling_param:
|
||||
return SamplingParams.from_optional(**diff_sampling_param)
|
||||
if self.default_sampling_params is None:
|
||||
self.default_sampling_params = (
|
||||
self.llm_engine.model_config.get_diff_sampling_param())
|
||||
if self.default_sampling_params:
|
||||
return SamplingParams.from_optional(**self.default_sampling_params)
|
||||
return SamplingParams()
|
||||
|
||||
@overload
|
||||
|
||||
Reference in New Issue
Block a user