[MM] Keep more metadata tensors on CPU (#52827)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-18 19:56:33 -07:00
committed by GitHub
parent aabc1a0a0b
commit deeeae75d0
38 changed files with 148 additions and 83 deletions
+7 -3
View File
@@ -254,8 +254,8 @@ def _qwen3asr_field_config(hf_inputs: Mapping[str, torch.Tensor]):
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
"audio", audio_feature_lengths, dim=1
),
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
feature_attention_mask=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
audio_feature_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
@@ -459,7 +459,11 @@ class Qwen3ASRForConditionalGeneration(
audio_input: Qwen2_5OmniAudioFeatureInputs,
) -> torch.Tensor:
input_features = audio_input["input_features"]
audio_feature_lengths = audio_input["audio_feature_lengths"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
)
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)