From 3b6c96a101651c78737b23b2bd8f1026c02f22e2 Mon Sep 17 00:00:00 2001 From: Sebastian Woo <43553500+woosebastian@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:19:12 -0700 Subject: [PATCH] [Bugfix][Pooling] Fix wrong scores for chunked prefill under torch.compile (#48901) Signed-off-by: seewoo Signed-off-by: wang.yuqi Co-authored-by: Claude Co-authored-by: wang.yuqi --- .../test_all_pooling_plus_chunked_prefill.py | 51 +++++++++++++++++++ vllm/v1/worker/gpu_model_runner.py | 1 + 2 files changed, 52 insertions(+) diff --git a/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py index 4119e1d5e0..251bc8c1d8 100644 --- a/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py +++ b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py @@ -5,6 +5,7 @@ import torch from transformers import AutoModel from tests.models.utils import check_embeddings_close +from tests.utils import VLLM_PATH from vllm import TokensPrompt from vllm.config import PoolerConfig @@ -53,3 +54,53 @@ def test_embed_models(hf_runner, vllm_runner, model: str): name_1="vllm", tol=1e-2, ) + + +_RERANKER_HF_OVERRIDES = { + "architectures": ["Qwen3ForSequenceClassification"], + "classifier_from_token": ["no", "yes"], + "is_original_qwen3_reranker": True, +} +_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja" + + +@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"]) +@torch.inference_mode +def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str): + """LAST-pooling score must not depend on whether the prompt is chunked. + + Regression test for a wrong-score bug where, under ``torch.compile`` (i.e. + not ``enforce_eager``), a query+document pair long enough to be split across + prefill chunks produced a corrupted last-token hidden state and thus a + wrong relevance score, while the same input scored correctly unchunked. + The existing pooling+chunked-prefill coverage runs ``enforce_eager=True`` + and so never exercised the compiled path. + """ + chat_template = _RERANKER_TEMPLATE.read_text() + query = "What organelle produces energy in the cell?" + # A long, matching document so query + doc exceeds the chunk size below. + document = ( + "The mitochondria is the powerhouse of the cell. It generates most of " + "the cell chemical energy through oxidative phosphorylation. " + ) * 400 + + def score_with(max_num_batched_tokens: int) -> float: + with vllm_runner( + model, + runner="pooling", + hf_overrides=_RERANKER_HF_OVERRIDES, + max_model_len=16384, + max_num_batched_tokens=max_num_batched_tokens, + enable_chunked_prefill=True, + enable_prefix_caching=False, + ) as vllm_model: + return vllm_model.score(query, document, chat_template=chat_template)[0] + + # Chunk size forces the prompt across several prefill chunks; the large + # budget keeps it in a single chunk as the reference. + chunked = score_with(2048) + unchunked = score_with(16384) + + assert chunked == pytest.approx(unchunked, abs=5e-2), ( + f"chunked score {chunked} diverged from unchunked {unchunked}" + ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index ea8980a6e5..5039979b29 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3437,6 +3437,7 @@ class GPUModelRunner( ) if raw_pooler_output is None or not any(finished_mask): + self._sync_device() model_runner_output.pooler_output = [None] * num_reqs return model_runner_output