[BugFix] Fix get tokenizer when using ray (#3301)
This commit is contained in:
@@ -65,7 +65,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
lora_request = self._maybe_get_lora(request)
|
||||
guided_decode_logits_processor = (
|
||||
await get_guided_decoding_logits_processor(
|
||||
request, self.engine.get_tokenizer()))
|
||||
request, await self.engine.get_tokenizer()))
|
||||
if guided_decode_logits_processor:
|
||||
if sampling_params.logits_processors is None:
|
||||
sampling_params.logits_processors = []
|
||||
|
||||
@@ -126,7 +126,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
lora_request = self._maybe_get_lora(request)
|
||||
guided_decode_logit_processor = (
|
||||
await get_guided_decoding_logits_processor(
|
||||
request, self.engine.get_tokenizer()))
|
||||
request, await self.engine.get_tokenizer()))
|
||||
if guided_decode_logit_processor is not None:
|
||||
if sampling_params.logits_processors is None:
|
||||
sampling_params.logits_processors = []
|
||||
|
||||
Reference in New Issue
Block a user