[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -63,36 +63,32 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
|
||||
# Baseline: no cache, always processes from scratch.
|
||||
baseline_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config, cache=None
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Sender cache: on a cache hit returns (None, prompt_updates) for each
|
||||
# item, setting mm_kwargs["video"] = [None] – the exact condition that
|
||||
# triggered the original bug.
|
||||
sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
|
||||
cached_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config, cache=sender_cache
|
||||
)
|
||||
|
||||
video_token_id = baseline_processor.info.get_hf_config().video_token_id
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
mm_data = create_mm_data(num_videos)
|
||||
hf_processor_mm_kwargs = {"use_audio_in_video": True}
|
||||
|
||||
def run(processor):
|
||||
def run(processor, cache):
|
||||
return processor(
|
||||
[video_token_id] * num_videos,
|
||||
mm_items=baseline_processor.info.parse_mm_data(mm_data),
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
cache=cache,
|
||||
)["prompt_token_ids"]
|
||||
|
||||
baseline_ids = run(baseline_processor)
|
||||
baseline_ids = run(processor, cache=None)
|
||||
|
||||
# First call on the sender-cache processor: cache miss.
|
||||
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
|
||||
# detected normally from the item data.
|
||||
first_ids = run(cached_processor)
|
||||
first_ids = run(processor, cache=sender_cache)
|
||||
assert first_ids == baseline_ids, (
|
||||
"Cache-miss call produced different prompt_token_ids than baseline.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
@@ -103,7 +99,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
|
||||
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
|
||||
# not detected, yielding wrong token ids.
|
||||
second_ids = run(cached_processor)
|
||||
second_ids = run(processor, cache=sender_cache)
|
||||
assert second_ids == baseline_ids, (
|
||||
"Cache-hit call produced different prompt_token_ids than baseline.\n"
|
||||
"This is the regression introduced when use_audio_in_video detection\n"
|
||||
@@ -123,7 +119,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
@@ -72,6 +72,8 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
|
||||
|
||||
_IGNORE_MM_KEYS = {
|
||||
# Dithering causes minor divergence
|
||||
"cohere_asr": {"input_features"},
|
||||
# In Ultravox, the audio_features can be different depending on padding
|
||||
# The slight difference should not be a problem though, since
|
||||
# attention_mask lets us ignore the difference.
|
||||
@@ -286,8 +288,7 @@ def _test_processing_correctness(
|
||||
}
|
||||
)
|
||||
|
||||
baseline_processor = factories.build_processor(ctx, cache=None)
|
||||
cached_processor = factories.build_processor(ctx, cache=cache)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
@@ -336,41 +337,42 @@ def _test_processing_correctness(
|
||||
_test_processing_correctness_one(
|
||||
model_config,
|
||||
mm_data,
|
||||
baseline_processor,
|
||||
cached_processor,
|
||||
processor,
|
||||
batch_idx,
|
||||
hit_rate,
|
||||
num_batches,
|
||||
simplify_rate,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_correctness_one(
|
||||
model_config: ModelConfig,
|
||||
mm_data: MultiModalDataDict,
|
||||
baseline_processor: BaseMultiModalProcessor,
|
||||
cached_processor: BaseMultiModalProcessor,
|
||||
processor: BaseMultiModalProcessor,
|
||||
batch_idx: int,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
cache: MultiModalProcessorOnlyCache,
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
token_prompt = get_token_prompt(baseline_processor, mm_data)
|
||||
mm_items = baseline_processor.info.parse_mm_data(mm_data)
|
||||
token_prompt = get_token_prompt(processor, mm_data)
|
||||
mm_items = processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
baseline_tokenized_result = baseline_processor(
|
||||
baseline_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cached_tokenized_result = cached_processor(
|
||||
cached_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
|
||||
@@ -221,7 +221,7 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
|
||||
frames = np.stack([np.asarray(image)] * 2)
|
||||
@@ -230,15 +230,16 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
{"image": image, "video": [(frames, metadata)]}
|
||||
)
|
||||
|
||||
def process(mm_kwargs):
|
||||
def process(mm_kwargs, cache):
|
||||
return processor(
|
||||
hf_processor.image_token + hf_processor.video_token,
|
||||
mm_items,
|
||||
mm_uuid_items={"video": [video_uuid]},
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
baseline = process({})
|
||||
baseline = process({}, cache=None)
|
||||
if kwargs_on_init:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
@@ -247,12 +248,12 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
request_kwargs = {} if kwargs_on_init else kwargs
|
||||
process(request_kwargs)
|
||||
cached = process(request_kwargs)
|
||||
process(request_kwargs, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs)
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
def pixels(result, modality, field):
|
||||
return result["mm_kwargs"][modality][0][field].data
|
||||
|
||||
@@ -23,11 +23,9 @@ def test_profiling(model_id: str, max_model_len: int):
|
||||
model_config_kwargs=model_config_kwargs,
|
||||
limit_mm_per_prompt=mm_counts,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
|
||||
ctx.model_config,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
|
||||
|
||||
hf_config = ctx.get_hf_config(Llama4Config)
|
||||
image_size = hf_config.vision_config.image_size
|
||||
|
||||
@@ -168,7 +168,6 @@ def _build_moss_audio_processor(cache=None):
|
||||
MossAudioMultiModalProcessor(
|
||||
info,
|
||||
MossAudioDummyInputsBuilder(info),
|
||||
cache=cache,
|
||||
),
|
||||
ctx,
|
||||
)
|
||||
@@ -259,21 +258,24 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
|
||||
)
|
||||
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
cached_processor, _ = _build_moss_audio_processor(cache=cache)
|
||||
cached_processor, _ = _build_moss_audio_processor()
|
||||
cached_text_miss = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_text_hit = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_token_hit = cached_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
|
||||
|
||||
@@ -26,7 +26,6 @@ def test_jina_vl_processing_order() -> None:
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
@@ -37,6 +36,7 @@ def test_jina_vl_processing_order() -> None:
|
||||
return processor(
|
||||
placeholder * len(images),
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
query = process([query_image])
|
||||
|
||||
@@ -255,7 +255,7 @@ def test_model_tensor_schema(model_id: str):
|
||||
for modality, count in limit_mm_per_prompt.items()
|
||||
}
|
||||
)
|
||||
processor = factories.build_processor(ctx, cache=None)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
with initialize_dummy_model(model_cls, model_config) as model:
|
||||
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
|
||||
|
||||
@@ -245,6 +245,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
prompt="<image>\nWhat is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
first, second = process(), process()
|
||||
|
||||
@@ -48,4 +48,4 @@ def create_cached_processor(model_id: str, processor_cls):
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
|
||||
|
||||
Reference in New Issue
Block a user