[LoRA][Gemma4] Support vision tower LoRA (#42662)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: linitra24 <Joy25810@foxmail.com> Signed-off-by: linitra24 <joy25810@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
linitra24
parent
8e1131e1d1
commit
95c9144424
@@ -48,3 +48,4 @@ steps:
|
||||
- pytest -v -s -x lora/test_olmoe_tp.py
|
||||
- pytest -v -s -x lora/test_gptoss_tp.py
|
||||
- pytest -v -s -x lora/test_qwen35_densemodel_lora.py
|
||||
- pytest -v -s -x lora/test_gemma4_tp.py
|
||||
|
||||
@@ -523,7 +523,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
|
||||
| `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>E+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
|
||||
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
|
||||
| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I<sup>+</sup> + V + A<sup>*</sup> | `google/gemma-4-E2B-it`, etc. | | ✅︎ |
|
||||
| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I<sup>+</sup> + V + A<sup>*</sup> | `google/gemma-4-E2B-it`, etc. | ✅︎ | ✅︎ |
|
||||
| `Gemma4UnifiedForConditionalGeneration` | Gemma 4 Unified | T + I<sup>+</sup> + V + A | `google/gemma-4-12B-it`, etc. | | ✅︎ |
|
||||
| `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ |
|
||||
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -253,6 +253,11 @@ def qwen3vl_vision_lora_files():
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gemma4_vision_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_meowing_lora_files():
|
||||
"""Download Qwen3 Meow LoRA files once per test session."""
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# NOTE To avoid overloading the CI pipeline, this test script will not
|
||||
# be triggered on CI and is primarily intended for local testing and verification.
|
||||
|
||||
import vllm
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
from ..utils import multi_gpu_test
|
||||
|
||||
MODEL_PATH = "google/gemma-4-E2B-it"
|
||||
|
||||
PROMPT_TEMPLATE = """<bos><|turn>user
|
||||
<|image|>What is in the image?<turn|>
|
||||
<|turn>model
|
||||
"""
|
||||
|
||||
TEST_IMAGES = [
|
||||
ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom"),
|
||||
]
|
||||
|
||||
EXPECTED_OUTPUTS_VISION = [
|
||||
"A red stop sign stands prominently in the foreground.",
|
||||
"A majestic skyscraper stands tall, partially obscured by a vibrant "
|
||||
"canopy of cherry blossoms, against a clear blue sky.",
|
||||
]
|
||||
|
||||
|
||||
def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
|
||||
prompts = [
|
||||
{
|
||||
"prompt": PROMPT_TEMPLATE,
|
||||
"multi_modal_data": {"image": asset.pil_image},
|
||||
}
|
||||
for asset in TEST_IMAGES
|
||||
]
|
||||
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=128)
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
|
||||
)
|
||||
|
||||
generated_texts = [output.outputs[0].text.strip() for output in outputs]
|
||||
for generated, expected in zip(generated_texts, EXPECTED_OUTPUTS_VISION):
|
||||
assert generated.startswith(expected), (
|
||||
f"Generated text {generated!r} does not match expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_lora(gemma4_vision_lora_files):
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enable_lora=True,
|
||||
enforce_eager=True,
|
||||
max_loras=4,
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_gemma4_lora_tp2(gemma4_vision_lora_files):
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
enable_lora=True,
|
||||
max_loras=4,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=2,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_gemma4_lora_tp4(gemma4_vision_lora_files):
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
enable_lora=True,
|
||||
max_loras=4,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=4,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_cache_gb=0,
|
||||
enable_tower_connector_lora=True,
|
||||
)
|
||||
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
|
||||
@@ -380,9 +380,13 @@ def test_select_common_block_size_no_valid_option():
|
||||
|
||||
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
model = Mock()
|
||||
model.get_num_mm_encoder_tokens.side_effect = lambda num_embeds: num_embeds + 1
|
||||
model.get_mm_lora_token_counts.side_effect = (
|
||||
lambda *, modality, mm_kwargs, num_mm_embeds: (
|
||||
num_mm_embeds + 1,
|
||||
num_mm_embeds + 11,
|
||||
)
|
||||
)
|
||||
model.get_mm_mapping.return_value = SimpleNamespace(connector=True)
|
||||
model.get_num_mm_connector_tokens.side_effect = lambda num_tokens: num_tokens + 10
|
||||
|
||||
lora_manager = Mock()
|
||||
lora_manager.supports_tower_connector_lora.return_value = True
|
||||
|
||||
@@ -247,8 +247,17 @@ class LoRAModelManager:
|
||||
|
||||
mm_budget = MultiModalBudget(vllm_config, mm_registry)
|
||||
limit_per_prompt = max(mm_budget.mm_max_items_per_prompt.values())
|
||||
num_encoder_tokens = self.model.get_num_mm_encoder_tokens(
|
||||
mm_budget.get_encoder_budget()
|
||||
max_lora_tokens = mm_budget.get_encoder_budget()
|
||||
lora_token_counts_by_modality = [
|
||||
self.model.get_mm_lora_token_counts(
|
||||
modality=modality,
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=max_lora_tokens,
|
||||
)
|
||||
for modality in mm_budget.mm_max_toks_per_item
|
||||
]
|
||||
num_encoder_tokens = max(
|
||||
tower_tokens for tower_tokens, _ in lora_token_counts_by_modality
|
||||
)
|
||||
|
||||
# Tower wrappers
|
||||
@@ -263,10 +272,15 @@ class LoRAModelManager:
|
||||
|
||||
# Use wrapper for connector if present.
|
||||
if self.mm_mapping.connector:
|
||||
if hasattr(self.model, "get_num_mm_connector_tokens"):
|
||||
connector_tokens = self.model.get_num_mm_connector_tokens(
|
||||
num_encoder_tokens
|
||||
)
|
||||
connector_tokens = max(
|
||||
(
|
||||
connector_tokens
|
||||
for _, connector_tokens in lora_token_counts_by_modality
|
||||
if connector_tokens is not None
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
if connector_tokens is not None:
|
||||
connector_punica_wrapper = get_punica_wrapper(
|
||||
connector_tokens,
|
||||
max_batches=self.max_num_seqs * limit_per_prompt,
|
||||
|
||||
@@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration(
|
||||
self.multimodal_config = multimodal_config
|
||||
self.model_dtype = vllm_config.model_config.dtype
|
||||
self.vllm_config = vllm_config
|
||||
lora_config = vllm_config.lora_config
|
||||
self._enable_mm_lora = bool(
|
||||
lora_config is not None and lora_config.enable_tower_connector_lora
|
||||
)
|
||||
|
||||
# Only quantize towers when the quant method supports their
|
||||
# dimensions. BNB/torchao handle arbitrary sizes; other methods
|
||||
@@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
Groups images by patch count (resolution bucket) so each
|
||||
encoder call processes a uniform-shape batch with no
|
||||
cross-resolution padding. Pooling and projection are then
|
||||
applied over a single concatenated tensor for all images.
|
||||
cross-resolution padding. With MM LoRA enabled, all images are
|
||||
padded into one batch so the encoder call matches the tower mapping.
|
||||
"""
|
||||
pixel_values = image_input["pixel_values"]
|
||||
pixel_position_ids = image_input["pixel_position_ids"]
|
||||
@@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration(
|
||||
if isinstance(pixel_values, list)
|
||||
else pixel_values.shape[0]
|
||||
)
|
||||
pool_position_ids = pixel_position_ids
|
||||
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
|
||||
if self._enable_mm_lora:
|
||||
max_soft_tokens = vision_cfg.default_output_length
|
||||
mm_processor_kwargs = getattr(
|
||||
getattr(self, "multimodal_config", None),
|
||||
"mm_processor_kwargs",
|
||||
None,
|
||||
)
|
||||
if isinstance(mm_processor_kwargs, Mapping):
|
||||
value, _ = _get_max_soft_tokens(mm_processor_kwargs)
|
||||
if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS:
|
||||
max_soft_tokens = value
|
||||
|
||||
max_patches = max_soft_tokens * pooling_k2
|
||||
padded_position_ids: list[torch.Tensor] = []
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
num_patches = pv.shape[0]
|
||||
if num_patches > max_patches:
|
||||
raise ValueError(
|
||||
f"Image {idx} has {num_patches} patches, which exceeds "
|
||||
f"the MM LoRA patch limit of {max_patches}."
|
||||
)
|
||||
|
||||
pad_len = max_patches - num_patches
|
||||
pv = torch.cat(
|
||||
(pv, pv.new_zeros((pad_len, *pv.shape[1:]))),
|
||||
dim=0,
|
||||
)
|
||||
pp = torch.cat(
|
||||
(pp, pp.new_full((pad_len, *pp.shape[1:]), -1)),
|
||||
dim=0,
|
||||
)
|
||||
buckets.setdefault(max_patches, []).append((idx, pv, pp))
|
||||
padded_position_ids.append(pp)
|
||||
pool_position_ids = padded_position_ids
|
||||
else:
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
|
||||
|
||||
# Encode each resolution bucket in memory-safe chunks. Re-read
|
||||
# free memory per bucket because the previous bucket's encoder
|
||||
# pass has already allocated activations we should account for.
|
||||
last_hidden_states_map: dict[int, torch.Tensor] = {}
|
||||
for patches, items in buckets.items():
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
len(items),
|
||||
self._encoder_chunk(
|
||||
patches, free, total, vision_cfg.position_embedding_size
|
||||
),
|
||||
)
|
||||
if self._enable_mm_lora:
|
||||
max_batch_size = len(items)
|
||||
else:
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
len(items),
|
||||
self._encoder_chunk(
|
||||
patches, free, total, vision_cfg.position_embedding_size
|
||||
),
|
||||
)
|
||||
|
||||
for chunk_idx in range(0, len(items), max_batch_size):
|
||||
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
|
||||
@@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration(
|
||||
output_length = chunk_hidden.shape[0] // pooling_k2
|
||||
|
||||
single_hidden = chunk_hidden.unsqueeze(0)
|
||||
single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0)
|
||||
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
|
||||
padding_positions = (single_pos_ids == -1).all(dim=-1)
|
||||
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
@@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration(
|
||||
tower_model=tower_models,
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
if modality in ("image", "video"):
|
||||
vision_config = self.config.vision_config
|
||||
pooling_k2 = vision_config.pooling_kernel_size**2
|
||||
|
||||
if modality == "image":
|
||||
pixel_values_key = "pixel_values"
|
||||
max_soft_tokens = vision_config.default_output_length
|
||||
mm_processor_kwargs = getattr(
|
||||
getattr(self, "multimodal_config", None),
|
||||
"mm_processor_kwargs",
|
||||
None,
|
||||
)
|
||||
if isinstance(mm_processor_kwargs, Mapping):
|
||||
val, _ = _get_max_soft_tokens(mm_processor_kwargs)
|
||||
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
|
||||
max_soft_tokens = val
|
||||
else:
|
||||
pixel_values_key = "pixel_values_videos"
|
||||
max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS
|
||||
|
||||
tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None
|
||||
connector_tokens = num_mm_embeds
|
||||
if tower_tokens is None and mm_kwargs is not None:
|
||||
field = mm_kwargs.get(pixel_values_key)
|
||||
if field is not None:
|
||||
data = field.data
|
||||
if isinstance(data, torch.Tensor) and data.ndim >= 2:
|
||||
tower_tokens = int(math.prod(data.shape[:-1]))
|
||||
|
||||
if tower_tokens is None:
|
||||
min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS)
|
||||
tower_tokens = (
|
||||
math.ceil(num_mm_embeds / min_soft_tokens)
|
||||
* max_soft_tokens
|
||||
* pooling_k2
|
||||
)
|
||||
|
||||
if modality == "audio":
|
||||
tower_tokens = num_mm_embeds
|
||||
connector_tokens = num_mm_embeds
|
||||
|
||||
if mm_kwargs is not None:
|
||||
field = mm_kwargs.get("input_features_padded")
|
||||
if field is not None:
|
||||
data = field.data
|
||||
if isinstance(data, torch.Tensor) and data.ndim >= 2:
|
||||
batch_size = math.prod(data.shape[:-2])
|
||||
audio_tokens = batch_size * math.ceil(data.shape[-2] / 4)
|
||||
tower_tokens = audio_tokens
|
||||
connector_tokens = audio_tokens
|
||||
|
||||
return tower_tokens, connector_tokens
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
|
||||
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""
|
||||
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
del modality, mm_kwargs
|
||||
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
|
||||
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
|
||||
return (
|
||||
num_encoder_tokens,
|
||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||
)
|
||||
|
||||
@overload
|
||||
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ def set_active_mm_loras(
|
||||
token_lora_mapping: list[int] = []
|
||||
lora_requests = set()
|
||||
encoder_token_counts: list[int] = []
|
||||
connector_token_counts: list[int] = []
|
||||
|
||||
# iterate through images
|
||||
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
|
||||
@@ -42,10 +43,17 @@ def set_active_mm_loras(
|
||||
# iterate through visual tokens
|
||||
for mm_input_id in encoder_input_ids:
|
||||
pos_info = mm_features[mm_input_id].mm_position
|
||||
num_tokens = model.get_num_mm_encoder_tokens(pos_info.get_num_embeds())
|
||||
|
||||
tower_tokens, connector_tokens = model.get_mm_lora_token_counts(
|
||||
modality=mm_features[mm_input_id].modality,
|
||||
mm_kwargs=mm_features[mm_input_id].data,
|
||||
num_mm_embeds=pos_info.get_num_embeds(),
|
||||
)
|
||||
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * num_tokens)
|
||||
encoder_token_counts.append(num_tokens)
|
||||
token_lora_mapping.extend([lora_id] * tower_tokens)
|
||||
encoder_token_counts.append(tower_tokens)
|
||||
connector_token_counts.append(connector_tokens)
|
||||
|
||||
if lora_id > 0:
|
||||
lora_request = lora_state.lora_requests.get(req_id)
|
||||
@@ -69,19 +77,13 @@ def set_active_mm_loras(
|
||||
if (
|
||||
mm_mapping is None
|
||||
or not mm_mapping.connector
|
||||
or not hasattr(model, "get_num_mm_connector_tokens")
|
||||
or not all(count is not None for count in connector_token_counts)
|
||||
):
|
||||
return
|
||||
|
||||
connector_token_mapping = np.repeat(
|
||||
np.array(prompt_lora_mapping, dtype=np.int32),
|
||||
np.array(
|
||||
[
|
||||
model.get_num_mm_connector_tokens(num_tokens)
|
||||
for num_tokens in encoder_token_counts
|
||||
],
|
||||
dtype=np.int32,
|
||||
),
|
||||
np.array(connector_token_counts, dtype=np.int32),
|
||||
)
|
||||
lora_manager.set_active_adapters(
|
||||
lora_requests,
|
||||
|
||||
@@ -3123,18 +3123,24 @@ class GPUModelRunner(
|
||||
token_lora_mapping = []
|
||||
lora_requests = set()
|
||||
encoder_token_counts = []
|
||||
connector_token_counts = []
|
||||
|
||||
for req_id, pos_info in mm_lora_refs:
|
||||
for (req_id, pos_info), (modality, mm_item) in zip(
|
||||
mm_lora_refs,
|
||||
mm_kwargs,
|
||||
):
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
lora_id = int(self.input_batch.request_lora_mapping[req_idx])
|
||||
|
||||
# Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
|
||||
num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
|
||||
pos_info.get_num_embeds()
|
||||
tower_tokens, connector_tokens = self.model.get_mm_lora_token_counts( # type: ignore[attr-defined]
|
||||
modality=modality,
|
||||
mm_kwargs=mm_item,
|
||||
num_mm_embeds=pos_info.get_num_embeds(),
|
||||
)
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * num_tokens)
|
||||
encoder_token_counts.append(num_tokens)
|
||||
token_lora_mapping.extend([lora_id] * tower_tokens)
|
||||
encoder_token_counts.append(tower_tokens)
|
||||
connector_token_counts.append(connector_tokens)
|
||||
|
||||
if lora_id > 0:
|
||||
lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id)
|
||||
@@ -3162,16 +3168,11 @@ class GPUModelRunner(
|
||||
if (
|
||||
mm_mapping is not None
|
||||
and mm_mapping.connector
|
||||
and hasattr(self.model, "get_num_mm_connector_tokens")
|
||||
and all(count is not None for count in connector_token_counts)
|
||||
):
|
||||
post_op_counts = [
|
||||
self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
|
||||
for num_tokens in encoder_token_counts
|
||||
]
|
||||
|
||||
connector_token_mapping = np.repeat(
|
||||
np.array(prompt_lora_mapping, dtype=np.int32),
|
||||
np.array(post_op_counts, dtype=np.int32),
|
||||
np.array(connector_token_counts, dtype=np.int32),
|
||||
)
|
||||
connector_mapping = LoRAMapping(
|
||||
index_mapping=tuple(connector_token_mapping.tolist()),
|
||||
|
||||
Reference in New Issue
Block a user