[Frontend] Add video embeds input support (#54242)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
# Timestamps are part of the prompt replacement (each frame group's
|
||||
# "<X.X seconds>" text), so they size the placeholder range and must
|
||||
# travel with the grid when embeddings are delivered out of band.
|
||||
embedding_fields = {
|
||||
**Qwen2VLMultiModalDataParser.embedding_fields,
|
||||
"video": {
|
||||
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
|
||||
"timestamps": "metadata",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(Qwen3VLConfig)
|
||||
@@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return self.get_hf_processor(**kwargs).video_processor
|
||||
|
||||
def get_data_parser(self):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
return Qwen3VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
@@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
def get_video_repl(
|
||||
*,
|
||||
tokens_per_frame: list[int],
|
||||
timestamps: list[float | int],
|
||||
timestamps: list[float | int] | torch.Tensor,
|
||||
tokenizer: TokenizerLike,
|
||||
vision_start_token_id: int,
|
||||
vision_end_token_id: int,
|
||||
|
||||
Reference in New Issue
Block a user