[Misc] Make PlaceholderRange.get_num_embeds a method (#34035)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -33,7 +33,7 @@ def get_mm_max_toks_per_item(
|
||||
)
|
||||
|
||||
return {
|
||||
modality: sum(item.get_num_embeds for item in placeholders)
|
||||
modality: sum(item.get_num_embeds() for item in placeholders)
|
||||
for modality, placeholders in mm_inputs["mm_placeholders"].items()
|
||||
}
|
||||
|
||||
|
||||
@@ -199,7 +199,6 @@ class PlaceholderRange:
|
||||
def embeds_cumsum(self) -> torch.Tensor | None:
|
||||
return None if self.is_embed is None else self.is_embed.cumsum(dim=0)
|
||||
|
||||
@cached_property
|
||||
def get_num_embeds(self) -> int:
|
||||
if self.embeds_cumsum is None:
|
||||
return self.length
|
||||
|
||||
@@ -1100,7 +1100,7 @@ class Scheduler(SchedulerInterface):
|
||||
for i, mm_feature in enumerate(mm_features):
|
||||
start_pos = mm_feature.mm_position.offset
|
||||
num_encoder_tokens = mm_feature.mm_position.length
|
||||
num_encoder_embeds = mm_feature.mm_position.get_num_embeds
|
||||
num_encoder_embeds = mm_feature.mm_position.get_num_embeds()
|
||||
item_identifier = mm_feature.identifier
|
||||
|
||||
# The encoder output is needed if the two ranges overlap:
|
||||
|
||||
@@ -786,7 +786,7 @@ class InputProcessor:
|
||||
decoder_mm_positions = prompt_inputs["mm_placeholders"]
|
||||
for modality, mm_positions in decoder_mm_positions.items():
|
||||
for mm_position in mm_positions:
|
||||
embed_length = mm_position.get_num_embeds
|
||||
embed_length = mm_position.get_num_embeds()
|
||||
if embed_length > self.mm_encoder_cache_size:
|
||||
raise ValueError(
|
||||
f"The {prompt_type} prompt contains a(n) {modality} item "
|
||||
|
||||
+1
-1
@@ -260,7 +260,7 @@ class Request:
|
||||
|
||||
def get_num_encoder_embeds(self, input_id: int) -> int:
|
||||
assert input_id < len(self.mm_features)
|
||||
return self.mm_features[input_id].mm_position.get_num_embeds
|
||||
return self.mm_features[input_id].mm_position.get_num_embeds()
|
||||
|
||||
def record_event(
|
||||
self,
|
||||
|
||||
@@ -2326,7 +2326,7 @@ class GPUModelRunner(
|
||||
|
||||
# Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
|
||||
num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
|
||||
pos_info.get_num_embeds
|
||||
pos_info.get_num_embeds()
|
||||
)
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * num_tokens)
|
||||
|
||||
Reference in New Issue
Block a user