[Model] Add VaultGemma via Transformers modeling backend (#49803)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -464,6 +464,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor
|
||||
| `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ |
|
||||
| `SmolLM3ForCausalLM` | SmolLM3 | `HuggingFaceTB/SmolLM3-3B` | ✅︎ | ✅︎ |
|
||||
| `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | ✅︎ | ✅︎ |
|
||||
| `VaultGemmaForCausalLM` | VaultGemma | `google/vaultgemma-1b` | ✅︎ | ✅︎ |
|
||||
|
||||
!!! note
|
||||
Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
|
||||
|
||||
@@ -557,6 +557,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"TeleFLMForCausalLM": _HfExamplesInfo(
|
||||
"CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
|
||||
),
|
||||
"VaultGemmaForCausalLM": _HfExamplesInfo("google/vaultgemma-1b"),
|
||||
"Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
|
||||
"MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
|
||||
"MiMoV2FlashForCausalLM": _HfExamplesInfo(
|
||||
|
||||
@@ -102,10 +102,7 @@ from vllm.logger import init_logger, suppress_logging
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.plugins import load_general_plugins
|
||||
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
|
||||
from vllm.transformers_utils.config import (
|
||||
is_interleaved,
|
||||
maybe_override_with_speculators,
|
||||
)
|
||||
from vllm.transformers_utils.config import maybe_override_with_speculators
|
||||
from vllm.transformers_utils.repo_utils import get_model_path
|
||||
from vllm.transformers_utils.utils import is_cloud_storage
|
||||
from vllm.utils.argparse_utils import (
|
||||
@@ -1908,7 +1905,8 @@ class EngineArgs:
|
||||
self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
|
||||
self._set_default_reasoning_config_args()
|
||||
sliding_window: int | None = None
|
||||
if not is_interleaved(model_config.hf_text_config):
|
||||
layer_types = getattr(model_config.hf_text_config, "layer_types", None)
|
||||
if layer_types is None or all(lt == "sliding_attention" for lt in layer_types):
|
||||
# Only set CacheConfig.sliding_window if the model is all sliding
|
||||
# window. Otherwise CacheConfig.sliding_window will override the
|
||||
# global layers in interleaved sliding window models.
|
||||
|
||||
@@ -55,7 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import is_interleaved, set_default_rope_theta
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .interfaces import (
|
||||
@@ -345,7 +345,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
# TODO (@robertgshaw2): see if this can be moved out
|
||||
if is_interleaved(vllm_config.model_config.hf_text_config):
|
||||
if len(set(getattr(config, "layer_types", []))) > 1:
|
||||
assert config.max_window_layers == config.num_hidden_layers, (
|
||||
"Sliding window for some but all layers is not supported. "
|
||||
"This model uses sliding window but `max_window_layers` = {} "
|
||||
|
||||
@@ -659,6 +659,7 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
# Multimodal models
|
||||
"Emu3ForConditionalGeneration": (
|
||||
"transformers",
|
||||
|
||||
@@ -600,16 +600,6 @@ def is_encoder_decoder(config: PretrainedConfig) -> bool:
|
||||
return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
|
||||
|
||||
|
||||
def is_interleaved(config: PretrainedConfig) -> bool:
|
||||
"""
|
||||
Detect if the model with this config is used with interleaved attention.
|
||||
"""
|
||||
text_config = config.get_text_config()
|
||||
if layer_types := getattr(text_config, "layer_types", None):
|
||||
return len(set(layer_types)) > 1
|
||||
return False
|
||||
|
||||
|
||||
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
|
||||
"""
|
||||
Update kwargs for AutoConfig initialization based on model_type
|
||||
|
||||
Reference in New Issue
Block a user