[Model] Support EmbeddingGemma2 multimodal pooling architecture (#60254)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
7b665f7a58
commit
02b83919aa
@@ -282,6 +282,111 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingGemma2ModelConfig(Gemma4Config):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Gemma4Config.verify_and_update_config(vllm_config)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
attn_config = vllm_config.attention_config
|
||||
if attn_config.backend is None:
|
||||
attn_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
"EmbeddingGemma2: attention backend not specified; defaulting to "
|
||||
"TRITON_ATTN (validated backend for heterogeneous head dimensions)."
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"EmbeddingGemma2: using attention backend %s", attn_config.backend
|
||||
)
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
orig_len = getattr(model_config, "original_max_model_len", None)
|
||||
if orig_len in (None, -1):
|
||||
default_len = 8192
|
||||
# Check sentence_bert_config.json if available
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cfg_path = Path(model_config.model) / "sentence_bert_config.json"
|
||||
if cfg_path.exists():
|
||||
data = json.loads(cfg_path.read_text())
|
||||
if "max_seq_length" in data and isinstance(
|
||||
data["max_seq_length"], int
|
||||
):
|
||||
default_len = data["max_seq_length"]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if model_config.max_model_len > default_len:
|
||||
uncapped_len = model_config.max_model_len
|
||||
logger.info(
|
||||
"EmbeddingGemma2: max_model_len not explicitly set; capping from "
|
||||
"%d to %d.",
|
||||
model_config.max_model_len,
|
||||
default_len,
|
||||
)
|
||||
model_config.max_model_len = default_len
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
if scheduler_config is not None:
|
||||
new_batched_tokens = max(
|
||||
model_config.max_model_len, scheduler_config.max_num_seqs
|
||||
)
|
||||
if scheduler_config.max_num_batched_tokens >= uncapped_len:
|
||||
if scheduler_config.max_num_batched_tokens > uncapped_len:
|
||||
logger.warning(
|
||||
"EmbeddingGemma2: lowering explicitly set "
|
||||
"scheduler_config.max_num_batched_tokens from %d to %d "
|
||||
"to match capped max_model_len.",
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
new_batched_tokens,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"EmbeddingGemma2: lowering "
|
||||
"scheduler_config.max_num_batched_tokens from %d to %d "
|
||||
"to match capped max_model_len.",
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
new_batched_tokens,
|
||||
)
|
||||
scheduler_config.max_num_batched_tokens = new_batched_tokens
|
||||
if hasattr(vllm_config, "_set_compile_ranges"):
|
||||
vllm_config._set_compile_ranges()
|
||||
if scheduler_config.max_num_encoder_input_tokens >= uncapped_len:
|
||||
if scheduler_config.max_num_encoder_input_tokens > uncapped_len:
|
||||
logger.warning(
|
||||
"EmbeddingGemma2: lowering explicitly set "
|
||||
"scheduler_config.max_num_encoder_input_tokens "
|
||||
"from %d to %d to match capped max_model_len.",
|
||||
scheduler_config.max_num_encoder_input_tokens,
|
||||
new_batched_tokens,
|
||||
)
|
||||
scheduler_config.max_num_encoder_input_tokens = (
|
||||
new_batched_tokens
|
||||
)
|
||||
if scheduler_config.encoder_cache_size >= uncapped_len:
|
||||
if scheduler_config.encoder_cache_size > uncapped_len:
|
||||
logger.warning(
|
||||
"EmbeddingGemma2: lowering explicitly set "
|
||||
"scheduler_config.encoder_cache_size from %d to %d "
|
||||
"to match capped max_model_len.",
|
||||
scheduler_config.encoder_cache_size,
|
||||
new_batched_tokens,
|
||||
)
|
||||
scheduler_config.encoder_cache_size = new_batched_tokens
|
||||
if hasattr(scheduler_config, "verify_max_model_len"):
|
||||
scheduler_config.verify_max_model_len(
|
||||
model_config.max_model_len
|
||||
)
|
||||
elif model_config.max_model_len > 32768:
|
||||
logger.warning(
|
||||
"EmbeddingGemma2: max_model_len=%d is large; "
|
||||
"consider --max-model-len 8192 to bound encoder memory.",
|
||||
model_config.max_model_len,
|
||||
)
|
||||
|
||||
|
||||
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -1025,6 +1130,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
"EmbeddingGemma2Model": EmbeddingGemma2ModelConfig,
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
"FalconMambaForCausalLM": MambaModelConfig,
|
||||
"Gemma3TextModel": Gemma3TextModelConfig,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -246,6 +246,7 @@ _EMBEDDING_MODELS = {
|
||||
# [Multimodal]
|
||||
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
|
||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
|
||||
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
|
||||
"LlavaNextForConditionalGeneration": (
|
||||
"llava_next",
|
||||
|
||||
Reference in New Issue
Block a user