[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
+35 -7
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
InputProcessingContext,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {
"enable_time_marker",
"mel_config",
}
MOSS_AUDIO_PLACEHOLDER_TOKENS = (
MOSS_AUDIO_BOS_TOKEN,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_EOS_TOKEN,
)
def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None:
"""Register the audio placeholder tokens if the tokenizer lacks them.
The MOSS-Audio hub tokenizer does not define the audio placeholder
tokens (the reference HF processor patches them in at runtime). Left
as plain text, they merge with adjacent tokens under BPE, so prompts
no longer contain a stable token sequence for prompt updates to match.
"""
convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None)
add_tokens = getattr(tokenizer, "add_tokens", None)
if convert_tokens_to_ids is None or add_tokens is None:
return
unk_token_id = getattr(tokenizer, "unk_token_id", None)
missing_tokens = [
token
for token in MOSS_AUDIO_PLACEHOLDER_TOKENS
if convert_tokens_to_ids(token) in (None, unk_token_id)
]
if missing_tokens:
add_tokens(missing_tokens, special_tokens=True)
class MossAudioAudioInputs(TensorSchema):
@@ -1157,6 +1186,11 @@ class MossAudioProcessor:
class MossAudioProcessingInfo(BaseProcessingInfo):
def __init__(self, ctx: InputProcessingContext) -> None:
super().__init__(ctx)
if ctx.tokenizer is not None:
_ensure_moss_audio_placeholder_tokens(ctx.tokenizer)
def get_hf_config(self) -> MossAudioConfig:
config = self.ctx.get_hf_config()
if isinstance(config, MossAudioConfig):
@@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
mm_data["audio"] = audios
mm_kwargs = dict(mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
tok_kwargs = {
key: value
for key, value in tok_kwargs.items()
if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
}
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
dict(text=prompt, **mm_data),
dict(**tok_kwargs),
{},
)
def _get_mm_fields_config(