[BugFix] Fix get tokenizer when using ray (#3301)

This commit is contained in:
Roy
2024-03-10 19:17:16 -07:00
committed by GitHub
parent e4a28e5316
commit 9e8744a545
6 changed files with 23 additions and 7 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ class OpenAIServingChat(OpenAIServing):
lora_request = self._maybe_get_lora(request)
guided_decode_logits_processor = (
await get_guided_decoding_logits_processor(
request, self.engine.get_tokenizer()))
request, await self.engine.get_tokenizer()))
if guided_decode_logits_processor:
if sampling_params.logits_processors is None:
sampling_params.logits_processors = []
@@ -126,7 +126,7 @@ class OpenAIServingCompletion(OpenAIServing):
lora_request = self._maybe_get_lora(request)
guided_decode_logit_processor = (
await get_guided_decoding_logits_processor(
request, self.engine.get_tokenizer()))
request, await self.engine.get_tokenizer()))
if guided_decode_logit_processor is not None:
if sampling_params.logits_processors is None:
sampling_params.logits_processors = []