[Rust Frontend][gRPC] Add audio and video media inputs (#53760)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>

rust/src/chat/src/multimodal/expand.rs

@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
         }
 
         let replacement_len = replacement.tokens.len();
+        let structural_prefix = replacement.structural_prefix;
+        if structural_prefix > expanded.len() {
+            bail_multimodal!(
+                "placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
+                lane.placeholder_token,
+                structural_prefix,
+                expanded.len()
+            );
+        }
+        let range_len = replacement_len.saturating_add(structural_prefix);
         let is_embed = {
-            let mask = replacement
-                .tokens
-                .iter()
-                .map(|&token| token as u32 == lane.embed_token_id)
-                .collect::<Vec<_>>();
-            WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
+            let mut mask = Vec::with_capacity(range_len);
+            mask.extend(std::iter::repeat_n(false, structural_prefix));
+            mask.extend(
+                replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
+            );
+            WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
         };
 
-        let expanded_offset = expanded.len();
+        let expanded_offset = expanded.len() - structural_prefix;
         expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
         ranges.entry(lane.modality).or_default().push(PlaceholderRange {
             offset: expanded_offset,
-            length: replacement_len,
+            length: range_len,
             is_embed: Some(is_embed),
         });
     }
@@ -420,6 +430,29 @@ mod tests {
         assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
     }
 
+    #[test]
+    fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
+        const VISION_START_ID: u32 = 151652;
+        const VISION_END_ID: u32 = 151653;
+
+        let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
+        let replacement = PromptReplacement::repeated(
+            Modality::Video,
+            "<|video_pad|>",
+            QWEN3_VIDEO_PAD_ID as TokenId,
+            3,
+        )
+        .with_structural_prefix(1);
+        let prepared = vec![qwen3_video_prepared(vec![replacement])];
+
+        let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
+
+        let range = &ranges[&Modality::Video][0];
+        assert_eq!(range.offset, 1);
+        assert_eq!(range.length, 4);
+        assert_bool_mask(range, &[false, true, true, true]);
+    }
+
     const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
 
     fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {

rust/src/server/src/grpc/convert.rs

@@ -22,49 +22,66 @@ pub fn media_parts_from_request(
 ) -> Result<Vec<MediaContentPart>, Status> {
     let mut parts = Vec::with_capacity(media.len());
     for (index, item) in media.into_iter().enumerate() {
-        match item.modality() {
-            pb::Modality::Image => {}
-            pb::Modality::Unspecified => {
-                return Err(Status::invalid_argument(format!(
-                    "media[{index}].modality is required"
-                )));
-            }
-            other => {
-                return Err(Status::unimplemented(format!(
-                    "media[{index}].modality {other:?} is not supported by the gRPC service"
-                )));
-            }
+        let modality = item.modality();
+        if modality == pb::Modality::Unspecified {
+            return Err(Status::invalid_argument(format!(
+                "media[{index}].modality is required"
+            )));
         }
         let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
         let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
-        let part = match item.source {
-            Some(pb::media_item::Source::Url(url)) => {
-                validate_media_uri(index, "url", &url, &["http", "https"])?;
-                MediaContentPart::ImageUrl {
-                    url,
-                    detail: None,
-                    uuid,
-                }
+        let source = item.source.ok_or_else(|| {
+            Status::invalid_argument(format!("media[{index}].source is required"))
+        })?;
+        match &source {
+            pb::media_item::Source::Url(url) => {
+                validate_media_uri(index, "url", url, &["http", "https"])?;
             }
-            Some(pb::media_item::Source::DataUri(uri)) => {
-                validate_media_uri(index, "data_uri", &uri, &["data"])?;
-                MediaContentPart::ImageUrl {
-                    url: uri,
-                    detail: None,
-                    uuid,
-                }
+            pb::media_item::Source::DataUri(uri) => {
+                validate_media_uri(index, "data_uri", uri, &["data"])?;
             }
-            Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData {
-                data: bytes,
-                mime_type,
-                uuid,
+            pb::media_item::Source::RawBytes(_) => {}
+        }
+        let part = match (modality, source) {
+            (
+                pb::Modality::Image,
+                pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
+            ) => MediaContentPart::ImageUrl {
+                url,
                 detail: None,
+                uuid,
             },
-            None => {
-                return Err(Status::invalid_argument(format!(
-                    "media[{index}].source is required"
-                )));
+            (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
+                MediaContentPart::ImageData {
+                    data,
+                    mime_type,
+                    uuid,
+                    detail: None,
+                }
             }
+            (
+                pb::Modality::Video,
+                pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
+            ) => MediaContentPart::VideoUrl { url, uuid },
+            (pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
+                MediaContentPart::VideoData {
+                    data,
+                    mime_type,
+                    uuid,
+                }
+            }
+            (
+                pb::Modality::Audio,
+                pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
+            ) => MediaContentPart::AudioUrl { url, uuid },
+            (pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
+                MediaContentPart::AudioData {
+                    data,
+                    mime_type,
+                    uuid,
+                }
+            }
+            (pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
         };
         parts.push(part);
     }

tests/model_executor/test_qwen3_asr_mrope.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.model_executor.models.qwen3_asr import (
+    Qwen3ASRForConditionalGeneration,
+)
+from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
+
+
+def test_mrope_positions_with_stripped_audio_data() -> None:
+    input_tokens = list(range(7))
+    feature = MultiModalFeatureSpec(
+        data=None,
+        modality="audio",
+        identifier="audio",
+        mm_position=PlaceholderRange(offset=2, length=3),
+    )
+
+    positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
+        None, input_tokens, [feature]
+    )
+
+    expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
+    torch.testing.assert_close(positions, expected)
+    assert delta == 0

vllm/model_executor/models/qwen3_asr.py

@@ -575,13 +575,10 @@ class Qwen3ASRForConditionalGeneration(
         for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
             offset = mm_feature.mm_position.offset
 
-            # Get audio feature length from mm_feature data
-            audio_feature_length = mm_feature.data["audio_feature_lengths"].data
-            if isinstance(audio_feature_length, torch.Tensor):
-                audio_feature_length = audio_feature_length.item()
-            audio_len = _get_feat_extract_output_lengths(
-                torch.tensor(audio_feature_length)
-            ).item()
+            # The placeholder already contains one token per audio embedding.
+            # Use its length so M-RoPE can be reconstructed even when a covered
+            # prefix no longer carries the audio processor tensors.
+            audio_len = mm_feature.mm_position.length
 
             # Text segment before audio (includes audio_start token)
             text_len = offset - st