diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 6eb5ff5cc4..dcfbfd5b1b 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders(): assert manager.get_cached_input_ids(request) == set() +def test_free_encoder_inputs_defers_for_eagle_lookahead(): + """With EAGLE speculative decoding, the encoder input is retained one extra + position so the drafter's +1 look-ahead mm-embedding gather (which reads one + position past the target's computed range) still finds it cached. This is + the primary mechanism that prevents the drafter "Encoder cache miss"; the + worker-side token-embedding fallback is only a backstop.""" + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + # create_scheduler only builds ngram spec configs; force the eagle path that + # _free_encoder_inputs keys off (self.use_eagle). + scheduler.use_eagle = True + mm_positions = [[PlaceholderRange(offset=50, length=100)]] + request = create_requests( + num_requests=1, + num_tokens=250, + mm_positions=mm_positions, + )[0] + manager = scheduler.encoder_cache_manager + manager.allocate(request, 0) + mm_end = 150 # offset + length + + # Confirmed progress reaches the range end: without spec decode this frees + # (see test below), but the drafter's +1 look-ahead still needs it. + request.num_computed_tokens = mm_end + scheduler._free_encoder_inputs(request) + assert manager.get_cached_input_ids(request) == {0} + + # One position past the range end: the +1 look-ahead has now passed it. + request.num_computed_tokens = mm_end + 1 + scheduler._free_encoder_inputs(request) + assert manager.get_cached_input_ids(request) == set() + + def test_free_encoder_inputs_unchanged_without_spec_decode(): """Without speculative decoding, encoder inputs are freed as soon as num_computed_tokens passes the placeholder range, as before.""" diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py new file mode 100644 index 0000000000..79c13a2b96 --- /dev/null +++ b/tests/v1/worker/test_encoder_runner.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for EncoderRunner.gather_mm_embeddings (model runner V2). + +Covers the speculative-drafter encoder-cache handling: the drafter reads one +position ahead of the target model (``draft_lookahead``). The +1 look-ahead +feature past the processed boundary is used when its encoder output is present +and tolerated (token-embedding fallback) when it is not, while a miss within +the processed range still fails loudly. +""" + +import numpy as np +import pytest +import torch + +from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner + +pytestmark = pytest.mark.cpu_test + +HIDDEN = 4 + + +def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=None, + modality="image", + identifier=identifier, + mm_position=PlaceholderRange(offset=offset, length=length), + ) + + +def _make_runner( + features: list[MultiModalFeatureSpec], + cached: list[MultiModalFeatureSpec], +) -> EncoderRunner: + cache = EncoderCache() + cache.mm_features["req0"] = features + for f in cached: + length = f.mm_position.length + cache.encoder_outputs[f.identifier] = torch.arange( + length * HIDDEN, dtype=torch.float32 + ).reshape(length, HIDDEN) + return EncoderRunner( + model=None, # unused by gather_mm_embeddings + max_num_tokens=64, + hidden_size=HIDDEN, + encoder_cache=cache, + dtype=torch.float32, + device=torch.device("cpu"), + ) + + +def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int): + # Single prefilling request, computed_prefill=0, prefill_len large. + return runner.gather_mm_embeddings( + req_ids=["req0"], + total_num_scheduled_tokens=num_scheduled, + num_scheduled_tokens=np.array([num_scheduled]), + query_start_loc=np.array([0]), + prefill_lens=np.array([1000]), + computed_prefill_lens=np.array([0]), + draft_lookahead=draft_lookahead, + ) + + +def test_draft_lookahead_uses_boundary_feature_when_cached(): + """The drafter's +1 look-ahead can reach the feature at offset == + processed_end (the next chunk). When its encoder output is already cached + (the scheduler encoded it ahead), it is used for the look-ahead position + rather than ignored.""" + f0 = _feature("h0", offset=0, length=8) + f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end + runner = _make_runner([f0, f1], cached=[f0, f1]) + + mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1) + + # f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7. + assert len(mm_embeds) == 2 + assert bool(is_mm_embed[7]) + assert int(is_mm_embed.sum()) == 8 + + +def test_draft_lookahead_tolerates_missing_boundary_feature(): + """When the +1 look-ahead feature past the processed boundary is not yet + encoded, fall back to the token embedding (the draft token is verified by + the target) instead of raising.""" + f0 = _feature("h0", offset=0, length=8) + f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached + runner = _make_runner([f0, f1], cached=[f0]) + + mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1) + + # Only f0 is gathered; f1's boundary position falls back silently. + assert len(mm_embeds) == 1 + assert not bool(is_mm_embed[7]) + assert int(is_mm_embed.sum()) == 7 + + +def test_draft_lookahead_raises_on_interior_miss(): + """A miss for a feature within the processed range (not the look-ahead + boundary) is a real invariant violation and must fail loudly, even on the + drafter path.""" + f0 = _feature("h0", offset=0, length=8) # interior, within processed range + runner = _make_runner([f0], cached=[]) + + with pytest.raises(RuntimeError, match="Encoder cache miss"): + _gather(runner, num_scheduled=8, draft_lookahead=1) + + +def test_target_path_raises_on_encoder_cache_miss(): + """On the target path (no look-ahead) a miss is a real invariant + violation and must fail loudly.""" + f0 = _feature("h0", offset=0, length=8) + runner = _make_runner([f0], cached=[]) + + with pytest.raises(RuntimeError, match="Encoder cache miss"): + _gather(runner, num_scheduled=8, draft_lookahead=0) + + +@pytest.mark.parametrize("draft_lookahead", [0, 1]) +def test_multi_request_batch_gathers_per_request(draft_lookahead): + """Two prefilling requests in one batch: per-request query bounds must be + indexed by request, not applied as whole arrays.""" + a0 = _feature("a0", offset=0, length=8) + b0 = _feature("b0", offset=0, length=8) + cache = EncoderCache() + cache.mm_features["req0"] = [a0] + cache.mm_features["req1"] = [b0] + for f in (a0, b0): + cache.encoder_outputs[f.identifier] = torch.arange( + f.mm_position.length * HIDDEN, dtype=torch.float32 + ).reshape(f.mm_position.length, HIDDEN) + runner = EncoderRunner( + model=None, + max_num_tokens=64, + hidden_size=HIDDEN, + encoder_cache=cache, + dtype=torch.float32, + device=torch.device("cpu"), + ) + + mm_embeds, is_mm_embed = runner.gather_mm_embeddings( + req_ids=["req0", "req1"], + total_num_scheduled_tokens=16, + num_scheduled_tokens=np.array([8, 8]), + query_start_loc=np.array([0, 8]), + prefill_lens=np.array([1000, 1000]), + computed_prefill_lens=np.array([0, 0]), + draft_lookahead=draft_lookahead, + ) + + # Both requests contribute a feature; with the +1 skew each marks 7 of its + # 8 positions (the skew drops one), otherwise all 8. + assert len(mm_embeds) == 2 + assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16) diff --git a/tests/v1/worker/test_gpu_model_runner_mm_gather.py b/tests/v1/worker/test_gpu_model_runner_mm_gather.py new file mode 100644 index 0000000000..586acd5046 --- /dev/null +++ b/tests/v1/worker/test_gpu_model_runner_mm_gather.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for GPUModelRunner._gather_mm_embeddings (model runner V1). + +Mirrors tests/v1/worker/test_encoder_runner.py (the V2 runner): the EAGLE/MTP +drafter reads one position ahead of the target (shift_computed_tokens=1). The ++1 look-ahead feature past the processed boundary is used when its encoder +output is present and tolerated (token-embedding fallback) when it is not, +while a miss within the processed range still fails loudly. + +`_gather_mm_embeddings` only uses CPU-side state, so it is exercised against a +lightweight stub for `self` instead of a full (CUDA-only) runner. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange +from vllm.v1.worker.gpu_model_runner import GPUModelRunner + +pytestmark = pytest.mark.cpu_test + +HIDDEN = 4 + + +def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=None, + modality="image", + identifier=identifier, + mm_position=PlaceholderRange(offset=offset, length=length), + ) + + +def _gather(features, cached, *, num_scheduled, shift, num_computed=0): + encoder_cache = { + f.identifier: torch.arange( + f.mm_position.length * HIDDEN, dtype=torch.float32 + ).reshape(f.mm_position.length, HIDDEN) + for f in cached + } + req_state = SimpleNamespace(num_computed_tokens=num_computed, mm_features=features) + runner = SimpleNamespace( + input_batch=SimpleNamespace(req_ids=["req0"]), + requests={"req0": req_state}, + encoder_cache=encoder_cache, + is_multimodal_pruning_enabled=False, + uses_mrope=False, + ) + scheduler_output = SimpleNamespace( + total_num_scheduled_tokens=num_scheduled, + num_scheduled_tokens={"req0": num_scheduled}, + ) + return GPUModelRunner._gather_mm_embeddings( + runner, scheduler_output, shift_computed_tokens=shift + ) + + +def test_draft_shift_uses_boundary_feature_when_cached(): + """The drafter's +1 look-ahead reaches the feature at offset == + processed_end; when it is already cached it is used for the look-ahead + position rather than ignored.""" + f0 = _feature("h0", offset=0, length=8) + f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end + mm_embeds, is_mm_embed = _gather([f0, f1], [f0, f1], num_scheduled=8, shift=1) + + # f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7. + assert len(mm_embeds) == 2 + assert bool(is_mm_embed[7]) + assert int(is_mm_embed.sum()) == 8 + + +def test_draft_shift_tolerates_missing_boundary_feature(): + """When the +1 look-ahead feature past the processed boundary is not yet + encoded, fall back to the token embedding instead of raising.""" + f0 = _feature("h0", offset=0, length=8) + f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached + mm_embeds, is_mm_embed = _gather([f0, f1], [f0], num_scheduled=8, shift=1) + + assert len(mm_embeds) == 1 # only f0; f1's boundary position falls back + assert not bool(is_mm_embed[7]) + assert int(is_mm_embed.sum()) == 7 + + +def test_draft_shift_raises_on_interior_miss(): + """A miss for a feature within the processed range (not the look-ahead + boundary) is a real invariant violation, even on the drafter path.""" + f0 = _feature("h0", offset=0, length=8) # interior, within processed range + with pytest.raises(RuntimeError, match="Encoder cache miss"): + _gather([f0], [], num_scheduled=8, shift=1) + + +def test_target_path_raises_on_encoder_cache_miss(): + """On the target path (no shift) a miss is a real invariant violation.""" + f0 = _feature("h0", offset=0, length=8) + with pytest.raises(RuntimeError, match="Encoder cache miss"): + _gather([f0], [], num_scheduled=8, shift=0) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 9f7c8d74de..90d93a110c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1872,6 +1872,11 @@ class Scheduler(SchedulerInterface): if not cached_encoder_input_ids: return + # Defer the free by the drafter's look-ahead so an entry stays + # referenced until the drafter's +1 read has also passed it, mirroring + # the shift the encoder scheduling path applies. + spec_lookahead = 1 if self.use_eagle else 0 + # Here, we use list(set) to avoid modifying the set while iterating # over it. for input_id in list(cached_encoder_input_ids): @@ -1884,13 +1889,12 @@ class Scheduler(SchedulerInterface): # KVs have been calculated and cached already. self.encoder_cache_manager.free_encoder_input(request, input_id) elif ( - start_pos + num_tokens + start_pos + num_tokens + spec_lookahead <= request.num_computed_tokens - request.num_output_placeholders ): - # The encoder output is already processed and stored in the - # decoder's KV cache, and progress is far enough past the - # placeholder range that no pending draft-token rejection can - # roll num_computed_tokens back into it. + # Processed, stored in the decoder KV cache, and far enough past + # the placeholder range (plus the drafter's look-ahead) that no + # rejection or drafter gather can reference it. self.encoder_cache_manager.free_encoder_input(request, input_id) def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None: diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index d86a166fbb..f0e99fae1f 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -68,15 +68,19 @@ class EncoderRunner: query_start_loc: np.ndarray, prefill_lens: np.ndarray, computed_prefill_lens: np.ndarray, + draft_lookahead: int = 0, ) -> tuple[list[torch.Tensor], torch.Tensor]: - is_prefilling = (computed_prefill_lens < prefill_lens).tolist() - all_decode = not any(is_prefilling) - if all_decode: + if draft_lookahead: + computed_prefill_lens = computed_prefill_lens + draft_lookahead + + is_prefilling_np = computed_prefill_lens < prefill_lens + if not is_prefilling_np.any(): # All decode requests, so no need to gather any embeddings. return [], torch.zeros( total_num_scheduled_tokens, dtype=torch.bool, device=self.device ) + is_prefilling = is_prefilling_np.tolist() query_start = computed_prefill_lens.tolist() query_end = (computed_prefill_lens + num_scheduled_tokens).tolist() @@ -89,11 +93,12 @@ class EncoderRunner: # OPTIMIZATION: Skip decode requests. continue + cur_query_start = query_start[i] + cur_query_end = query_end[i] + mm_features = self.encoder_cache.mm_features[req_id] lo, hi = get_mm_features_in_window( - mm_features, - start=query_start[i], - end=query_end[i], + mm_features, start=cur_query_start, end=cur_query_end ) for idx in range(lo, hi): mm_feature = mm_features[idx] @@ -101,8 +106,8 @@ class EncoderRunner: start_pos = pos_info.offset num_encoder_tokens = pos_info.length - start_idx = max(query_start[i] - start_pos, 0) - end_idx = min(query_end[i] - start_pos, num_encoder_tokens) + start_idx = max(cur_query_start - start_pos, 0) + end_idx = min(cur_query_end - start_pos, num_encoder_tokens) assert start_idx < end_idx curr_embeds_start, curr_embeds_end = ( pos_info.get_embeds_indices_in_range(start_idx, end_idx) @@ -114,7 +119,13 @@ class EncoderRunner: mm_hash = mm_feature.identifier encoder_output = self.encoder_cache.encoder_outputs.get(mm_hash, None) - assert encoder_output is not None, f"Encoder cache miss for {mm_hash}." + if encoder_output is None: + # A feature starting at/after the processed boundary is only + # reached via the drafter's +1 look-ahead and might not be + # encoded yet; fall back to the token embedding for drafting. + if start_pos + draft_lookahead >= cur_query_end: + continue + raise RuntimeError(f"Encoder cache miss for {mm_hash}.") if (is_embed := pos_info.is_embed) is not None: is_embed = is_embed[start_idx:end_idx] @@ -122,7 +133,7 @@ class EncoderRunner: else: mm_embeds_item = encoder_output[start_idx:end_idx] - req_start_pos = query_start_loc[i] + start_pos - query_start[i] + req_start_pos = query_start_loc[i] + start_pos - cur_query_start is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= ( True if is_embed is None else is_embed ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 124eb10186..30ca2ddc56 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1409,8 +1409,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch.num_scheduled_tokens, input_batch.query_start_loc_np, input_batch.prefill_len_np, - # +1 to consider the skew in eagle - input_batch.num_computed_prefill_tokens_np + 1, + input_batch.num_computed_prefill_tokens_np, + # The EAGLE/MTP drafter reads one position ahead of the target. + draft_lookahead=1, ) # Postprocess results and update request states. diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 173e285ca8..74938a823d 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3152,7 +3152,16 @@ class GPUModelRunner( mm_hash = mm_feature.identifier encoder_output = self.encoder_cache.get(mm_hash, None) - assert encoder_output is not None, f"Encoder cache miss for {mm_hash}." + if encoder_output is None: + # A feature starting at/after the processed boundary is only + # reached via the drafter's +1 look-ahead and might not be + # encoded yet; fall back to the token embedding for drafting. + if ( + start_pos + >= req_state.num_computed_tokens + num_scheduled_tokens + ): + continue + raise RuntimeError(f"Encoder cache miss for {mm_hash}.") if (is_embed := pos_info.is_embed) is not None: is_embed = is_embed[start_idx:end_idx]