[Bugfix][Pooling] Fix wrong scores for chunked prefill under torch.compile (#48901)

Signed-off-by: seewoo <seewoo@ucsc.edu>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Sebastian Woo
2026-07-17 05:19:12 +00:00
committed by GitHub
co-authored by Claude wang.yuqi
parent 4d4e04f452
commit 3b6c96a101
2 changed files with 52 additions and 0 deletions
@@ -5,6 +5,7 @@ import torch
from transformers import AutoModel
from tests.models.utils import check_embeddings_close
from tests.utils import VLLM_PATH
from vllm import TokensPrompt
from vllm.config import PoolerConfig
@@ -53,3 +54,53 @@ def test_embed_models(hf_runner, vllm_runner, model: str):
name_1="vllm",
tol=1e-2,
)
_RERANKER_HF_OVERRIDES = {
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
@torch.inference_mode
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
"""LAST-pooling score must not depend on whether the prompt is chunked.
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
not ``enforce_eager``), a query+document pair long enough to be split across
prefill chunks produced a corrupted last-token hidden state and thus a
wrong relevance score, while the same input scored correctly unchunked.
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
and so never exercised the compiled path.
"""
chat_template = _RERANKER_TEMPLATE.read_text()
query = "What organelle produces energy in the cell?"
# A long, matching document so query + doc exceeds the chunk size below.
document = (
"The mitochondria is the powerhouse of the cell. It generates most of "
"the cell chemical energy through oxidative phosphorylation. "
) * 400
def score_with(max_num_batched_tokens: int) -> float:
with vllm_runner(
model,
runner="pooling",
hf_overrides=_RERANKER_HF_OVERRIDES,
max_model_len=16384,
max_num_batched_tokens=max_num_batched_tokens,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as vllm_model:
return vllm_model.score(query, document, chat_template=chat_template)[0]
# Chunk size forces the prompt across several prefill chunks; the large
# budget keeps it in a single chunk as the reference.
chunked = score_with(2048)
unchunked = score_with(16384)
assert chunked == pytest.approx(unchunked, abs=5e-2), (
f"chunked score {chunked} diverged from unchunked {unchunked}"
)
+1
View File
@@ -3437,6 +3437,7 @@ class GPUModelRunner(
)
if raw_pooler_output is None or not any(finished_mask):
self._sync_device()
model_runner_output.pooler_output = [None] * num_reqs
return model_runner_output