[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-31 03:20:47 +00:00
committed by GitHub
parent 5707355209
commit 555ea65e8c
8 changed files with 461 additions and 24 deletions
+15 -2
View File
@@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Timestamps are part of the prompt replacement (each frame group's
# "<X.X seconds>" text), so they size the placeholder range and must
# travel with the grid when embeddings are delivered out of band.
embedding_fields = {
**Qwen2VLMultiModalDataParser.embedding_fields,
"video": {
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
"timestamps": "metadata",
},
}
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Qwen3VLConfig)
@@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
return self.get_hf_processor(**kwargs).video_processor
def get_data_parser(self):
return Qwen2VLMultiModalDataParser(
return Qwen3VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
@@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
def get_video_repl(
*,
tokens_per_frame: list[int],
timestamps: list[float | int],
timestamps: list[float | int] | torch.Tensor,
tokenizer: TokenizerLike,
vision_start_token_id: int,
vision_end_token_id: int,