[Bugfix] Validate routed-expert prompt offsets before engine submission (#56844)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Bhushan Asati <bhushanasati25@gmail.com> Co-authored-by: Megha Agarwal <19240983+meghaagr13@users.noreply.github.com>
This commit is contained in:
co-authored by
Bhushan Asati
Megha Agarwal
parent
ef5f7cd119
commit
84e5f3edac
@@ -354,12 +354,21 @@ class InputProcessor:
|
||||
if isinstance(params, SamplingParams):
|
||||
# TODO: can we avoid cloning here in multiproc case?
|
||||
sampling_params = params.clone()
|
||||
prompt_len = length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds
|
||||
)
|
||||
if not 0 <= sampling_params.routed_experts_prompt_start <= prompt_len:
|
||||
raise VLLMValidationError(
|
||||
f"routed_experts_prompt_start must be between 0 and "
|
||||
f"the prompt length ({prompt_len}), inclusive.",
|
||||
parameter="routed_experts_prompt_start",
|
||||
value=sampling_params.routed_experts_prompt_start,
|
||||
)
|
||||
# If unset max tokens, then generate up to the max_model_len.
|
||||
if sampling_params.max_tokens is None:
|
||||
seq_len = length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds
|
||||
sampling_params.max_tokens = (
|
||||
self.model_config.max_model_len - prompt_len
|
||||
)
|
||||
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
|
||||
|
||||
sampling_params.update_from_generation_config(
|
||||
self.generation_config_fields,
|
||||
@@ -368,12 +377,7 @@ class InputProcessor:
|
||||
if self.tokenizer is not None:
|
||||
sampling_params.update_from_tokenizer(self.tokenizer)
|
||||
if sampling_params.trace_decode_token_ids:
|
||||
self._normalize_trace_replay_params(
|
||||
sampling_params,
|
||||
length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds
|
||||
),
|
||||
)
|
||||
self._normalize_trace_replay_params(sampling_params, prompt_len)
|
||||
else:
|
||||
pooling_params = params.clone()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user