[Bugfix][V1] Reject encoder-cache hits with mismatched embedding counts (#57696)
Signed-off-by: jackLei0901 <42642542+jackLei0901@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b44895cf93
commit
bbc4ddeca9
@@ -353,6 +353,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.encoder_cache_mismatch_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.return_sampling_mask = False
|
||||
|
||||
@@ -114,6 +114,7 @@ def test_eviction_when_cache_is_full():
|
||||
|
||||
# 'x' should have been evicted.
|
||||
assert "x" not in manager.cached
|
||||
assert "x" not in manager.num_embeds_by_hash
|
||||
assert "req1" not in manager.request_cached_ids
|
||||
assert "x" in manager.get_freed_mm_hashes()
|
||||
|
||||
@@ -273,6 +274,31 @@ def test_encoder_cache_with_is_embed_mask():
|
||||
assert savings_ratio == 12.5
|
||||
|
||||
|
||||
def test_encoder_cache_tracks_embedding_count_by_identifier():
|
||||
class MockRequestWithMask(MockRequest):
|
||||
def get_num_encoder_embeds(self, input_id: int) -> int:
|
||||
return self.mm_features[input_id].mm_position.get_num_embeds()
|
||||
|
||||
is_embed = torch.tensor([False, True, True, False, True, False])
|
||||
request = MockRequestWithMask("owner", ["img1"], [len(is_embed)])
|
||||
request.mm_features[0] = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier="img1",
|
||||
mm_position=PlaceholderRange(offset=0, length=6, is_embed=is_embed),
|
||||
)
|
||||
|
||||
manager = EncoderCacheManager(cache_size=10)
|
||||
manager.allocate(request, 0)
|
||||
assert manager.get_cached_num_encoder_embeds(request, 0) == 3
|
||||
|
||||
manager.free(request)
|
||||
assert manager.get_cached_num_encoder_embeds(request, 0) == 3
|
||||
|
||||
manager.reset()
|
||||
assert manager.get_cached_num_encoder_embeds(request, 0) is None
|
||||
|
||||
|
||||
def test_encoder_cache_mask_based_retrieval():
|
||||
class MockRequestWithMask(MockRequest):
|
||||
def get_num_encoder_embeds(self, input_id: int) -> int:
|
||||
@@ -336,6 +362,7 @@ def test_reset_clears_all_state():
|
||||
manager.reset()
|
||||
|
||||
assert len(manager.cached) == 0
|
||||
assert len(manager.num_embeds_by_hash) == 0
|
||||
assert len(manager.request_cached_ids) == 0
|
||||
assert len(manager.freeable) == 0
|
||||
assert len(manager.freed) == 0
|
||||
|
||||
@@ -3896,6 +3896,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.encoder_cache_mismatch_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.return_sampling_mask = False
|
||||
scheduler.recompute_kv_load_failures = False
|
||||
@@ -6089,6 +6090,201 @@ def test_unavailable_encoder_input_fails_the_request_as_retryable():
|
||||
assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR]
|
||||
|
||||
|
||||
def _create_mm_request_with_embedding_positions(
|
||||
request_id: str, identifier: str, is_embed: list[bool]
|
||||
) -> Request:
|
||||
return create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=len(is_embed),
|
||||
mm_hashes_list=[[identifier]],
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(
|
||||
offset=0,
|
||||
length=len(is_embed),
|
||||
is_embed=torch.tensor(is_embed),
|
||||
)
|
||||
]
|
||||
],
|
||||
req_ids=[request_id],
|
||||
)[0]
|
||||
|
||||
|
||||
def test_encoder_cache_accepts_matching_embed_count():
|
||||
"""An actively referenced entry is reused when embedding counts match."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
identifier = "reused-client-uuid"
|
||||
owner = _create_mm_request_with_embedding_positions(
|
||||
"owner", identifier, [True, True, True, False, False, False]
|
||||
)
|
||||
compatible = _create_mm_request_with_embedding_positions(
|
||||
"compatible", identifier, [True, False, True, False, True, False, False]
|
||||
)
|
||||
scheduler.encoder_cache_manager.allocate(owner, 0)
|
||||
scheduler.add_request(compatible)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
assert compatible.request_id in scheduler_output.num_scheduled_tokens
|
||||
assert compatible.request_id not in scheduler.encoder_cache_mismatch_reqs
|
||||
assert compatible.request_id not in scheduler_output.scheduled_encoder_inputs
|
||||
|
||||
|
||||
def test_encoder_cache_rejects_mismatched_embed_count(caplog_vllm):
|
||||
"""A completed request's freeable entry rejects an incompatible reuse."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
identifier = "reused-client-uuid"
|
||||
owner = _create_mm_request_with_embedding_positions(
|
||||
"owner", identifier, [True, True, True, False, False, False]
|
||||
)
|
||||
mismatched = _create_mm_request_with_embedding_positions(
|
||||
"mismatched", identifier, [True, True, True, True, False, False]
|
||||
)
|
||||
scheduler.encoder_cache_manager.allocate(owner, 0)
|
||||
scheduler.encoder_cache_manager.free(owner)
|
||||
freeable_slots_before = scheduler.encoder_cache_manager.num_freeable_slots
|
||||
scheduler.add_request(mismatched)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert not scheduler_output.num_scheduled_tokens
|
||||
assert mismatched.request_id in scheduler.encoder_cache_mismatch_reqs
|
||||
assert (
|
||||
"multimodal identifier 'reused-client-uuid' holds 3 embeddings"
|
||||
in caplog_vllm.text
|
||||
and "expects 4" in caplog_vllm.text
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(req_ids=[], req_id_to_index={}),
|
||||
)
|
||||
|
||||
assert mismatched.status == RequestStatus.FINISHED_ERROR
|
||||
assert not scheduler.encoder_cache_mismatch_reqs
|
||||
assert mismatched.request_id not in scheduler.requests
|
||||
assert scheduler.encoder_cache_manager.cached[identifier] == set()
|
||||
assert (
|
||||
mismatched.request_id not in scheduler.encoder_cache_manager.request_cached_ids
|
||||
)
|
||||
assert scheduler.encoder_cache_manager.freeable[identifier] == 3
|
||||
assert scheduler.encoder_cache_manager.num_freeable_slots == freeable_slots_before
|
||||
output = outputs[mismatched.client_index].outputs[0]
|
||||
assert output.request_id == mismatched.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
|
||||
healthy = create_requests(num_requests=1, req_ids=["healthy"], num_tokens=6)[0]
|
||||
scheduler.add_request(healthy)
|
||||
next_output = scheduler.schedule()
|
||||
assert healthy.request_id in next_output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_encoder_cache_embed_count_mismatch_restores_prior_hit_state():
|
||||
"""A prior hit is released when a later mismatch fails the request."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
manager = scheduler.encoder_cache_manager
|
||||
valid_identifier = "valid-reuse"
|
||||
mismatch_identifier = "mismatched-reuse"
|
||||
valid_owner = _create_mm_request_with_embedding_positions(
|
||||
"valid-owner", valid_identifier, [True, True, False]
|
||||
)
|
||||
mismatch_owner = _create_mm_request_with_embedding_positions(
|
||||
"mismatch-owner", mismatch_identifier, [True, True, True, False]
|
||||
)
|
||||
manager.allocate(valid_owner, 0)
|
||||
manager.free(valid_owner)
|
||||
manager.allocate(mismatch_owner, 0)
|
||||
manager.free(mismatch_owner)
|
||||
freeable_slots_before = manager.num_freeable_slots
|
||||
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=7,
|
||||
mm_hashes_list=[[valid_identifier, mismatch_identifier]],
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(
|
||||
offset=0,
|
||||
length=3,
|
||||
is_embed=torch.tensor([True, True, False]),
|
||||
),
|
||||
PlaceholderRange(
|
||||
offset=3,
|
||||
length=4,
|
||||
is_embed=torch.tensor([True, True, True, True]),
|
||||
),
|
||||
]
|
||||
],
|
||||
req_ids=["partially-mutated"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert not scheduler_output.num_scheduled_tokens
|
||||
assert request.request_id in scheduler.encoder_cache_mismatch_reqs
|
||||
assert request.request_id in manager.cached[valid_identifier]
|
||||
assert valid_identifier not in manager.freeable
|
||||
assert manager.num_freeable_slots == freeable_slots_before - 2
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(req_ids=[], req_id_to_index={}),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert request.request_id not in manager.request_cached_ids
|
||||
assert manager.cached[valid_identifier] == set()
|
||||
assert manager.freeable[valid_identifier] == 2
|
||||
assert manager.freeable[mismatch_identifier] == 3
|
||||
assert manager.num_freeable_slots == freeable_slots_before
|
||||
output = outputs[request.client_index].outputs[0]
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
|
||||
|
||||
def test_encoder_cache_rejects_mismatched_embed_count_within_request():
|
||||
"""One request cannot reuse an identifier with a different embed count."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
identifier = "reused-within-request"
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=8,
|
||||
mm_hashes_list=[[identifier, identifier]],
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(
|
||||
offset=0,
|
||||
length=4,
|
||||
is_embed=torch.tensor([True, True, False, False]),
|
||||
),
|
||||
PlaceholderRange(
|
||||
offset=4,
|
||||
length=4,
|
||||
is_embed=torch.tensor([True, True, True, False]),
|
||||
),
|
||||
]
|
||||
],
|
||||
req_ids=["mismatched-within-request"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert not scheduler_output.num_scheduled_tokens
|
||||
assert request.request_id in scheduler.encoder_cache_mismatch_reqs
|
||||
assert identifier not in scheduler.encoder_cache_manager.cached
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(req_ids=[], req_id_to_index={}),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
assert not scheduler.encoder_cache_mismatch_reqs
|
||||
assert request.request_id not in scheduler.requests
|
||||
output = outputs[request.client_index].outputs[0]
|
||||
assert output.request_id == request.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
|
||||
Reference in New Issue
Block a user