[Core] Gate prompt_embeds behind a feature flag (#17607)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -321,6 +321,10 @@ class ModelConfig:
|
||||
"""Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
`prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
output will contain token ids."""
|
||||
enable_prompt_embeds: bool = False
|
||||
"""If `True`, enables passing text embeddings as inputs via the
|
||||
`prompt_embeds` key. Note that enabling this will double the time required
|
||||
for graph compilation."""
|
||||
served_model_name: Optional[Union[str, list[str]]] = None
|
||||
"""The model name(s) used in the API. If multiple names are provided, the
|
||||
server will respond to any of the provided names. The model name in the
|
||||
|
||||
@@ -234,6 +234,7 @@ class EngineArgs:
|
||||
hf_config_path: Optional[str] = ModelConfig.hf_config_path
|
||||
task: TaskOption = ModelConfig.task
|
||||
skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
|
||||
enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
|
||||
tokenizer_mode: TokenizerMode = ModelConfig.tokenizer_mode
|
||||
trust_remote_code: bool = ModelConfig.trust_remote_code
|
||||
allowed_local_media_path: str = ModelConfig.allowed_local_media_path
|
||||
@@ -445,6 +446,8 @@ class EngineArgs:
|
||||
**model_kwargs["disable_cascade_attn"])
|
||||
model_group.add_argument("--skip-tokenizer-init",
|
||||
**model_kwargs["skip_tokenizer_init"])
|
||||
model_group.add_argument("--enable-prompt-embeds",
|
||||
**model_kwargs["enable_prompt_embeds"])
|
||||
model_group.add_argument("--served-model-name",
|
||||
**model_kwargs["served_model_name"])
|
||||
# This one is a special case because it is the
|
||||
@@ -874,6 +877,7 @@ class EngineArgs:
|
||||
disable_sliding_window=self.disable_sliding_window,
|
||||
disable_cascade_attn=self.disable_cascade_attn,
|
||||
skip_tokenizer_init=self.skip_tokenizer_init,
|
||||
enable_prompt_embeds=self.enable_prompt_embeds,
|
||||
served_model_name=self.served_model_name,
|
||||
limit_mm_per_prompt=self.limit_mm_per_prompt,
|
||||
use_async_output_proc=not self.disable_async_output_proc,
|
||||
|
||||
@@ -303,8 +303,11 @@ class InputPreprocessor:
|
||||
self,
|
||||
parsed_content: EmbedsPrompt,
|
||||
) -> EmbedsInputs:
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError("You must set `--enable-prompt-embeds` to input "
|
||||
"`prompt_embeds`.")
|
||||
if envs.VLLM_USE_V1:
|
||||
raise ValueError("prompt_embeds is only available in V0.")
|
||||
raise ValueError("`prompt_embeds` is only available in V0.")
|
||||
|
||||
prompt_embeds = parsed_content["prompt_embeds"]
|
||||
|
||||
|
||||
@@ -1565,7 +1565,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
# product.
|
||||
cudagraph_capture_sizes = self.vllm_config.compilation_config\
|
||||
.cudagraph_capture_sizes
|
||||
cudagraph_inputs_embeds = (True, False)
|
||||
cudagraph_inputs_embeds = ((
|
||||
True, False) if self.model_config.enable_prompt_embeds else
|
||||
(False, ))
|
||||
compilation_cases = itertools.product(
|
||||
cudagraph_capture_sizes,
|
||||
cudagraph_inputs_embeds,
|
||||
|
||||
Reference in New Issue
Block a user