[1/N] Initialize MM components in context managers (A-D) (#32632)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -460,20 +460,21 @@ class AudioFlamingo3ForConditionalGeneration(
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
self.audio_tower = AudioFlamingo3Encoder(
|
||||
config.audio_config,
|
||||
)
|
||||
self.multi_modal_projector = AudioFlamingo3MultiModalProjector(config)
|
||||
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["Qwen2ForCausalLM"],
|
||||
)
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio_tower = AudioFlamingo3Encoder(
|
||||
config.audio_config,
|
||||
)
|
||||
self.multi_modal_projector = AudioFlamingo3MultiModalProjector(config)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["Qwen2ForCausalLM"],
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
@@ -599,9 +600,6 @@ class AudioFlamingo3ForConditionalGeneration(
|
||||
current_idx += count
|
||||
return tuple(grouped_embeddings)
|
||||
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.language_model
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
if audio_input is None:
|
||||
|
||||
Reference in New Issue
Block a user