[Bugfix][Pooling] Fix wrong scores for chunked prefill under torch.compile (#48901)
Signed-off-by: seewoo <seewoo@ucsc.edu> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude
wang.yuqi
parent
4d4e04f452
commit
3b6c96a101
@@ -5,6 +5,7 @@ import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
@@ -53,3 +54,53 @@ def test_embed_models(hf_runner, vllm_runner, model: str):
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
_RERANKER_HF_OVERRIDES = {
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
|
||||
@torch.inference_mode
|
||||
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
|
||||
"""LAST-pooling score must not depend on whether the prompt is chunked.
|
||||
|
||||
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
|
||||
not ``enforce_eager``), a query+document pair long enough to be split across
|
||||
prefill chunks produced a corrupted last-token hidden state and thus a
|
||||
wrong relevance score, while the same input scored correctly unchunked.
|
||||
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
|
||||
and so never exercised the compiled path.
|
||||
"""
|
||||
chat_template = _RERANKER_TEMPLATE.read_text()
|
||||
query = "What organelle produces energy in the cell?"
|
||||
# A long, matching document so query + doc exceeds the chunk size below.
|
||||
document = (
|
||||
"The mitochondria is the powerhouse of the cell. It generates most of "
|
||||
"the cell chemical energy through oxidative phosphorylation. "
|
||||
) * 400
|
||||
|
||||
def score_with(max_num_batched_tokens: int) -> float:
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
hf_overrides=_RERANKER_HF_OVERRIDES,
|
||||
max_model_len=16384,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as vllm_model:
|
||||
return vllm_model.score(query, document, chat_template=chat_template)[0]
|
||||
|
||||
# Chunk size forces the prompt across several prefill chunks; the large
|
||||
# budget keeps it in a single chunk as the reference.
|
||||
chunked = score_with(2048)
|
||||
unchunked = score_with(16384)
|
||||
|
||||
assert chunked == pytest.approx(unchunked, abs=5e-2), (
|
||||
f"chunked score {chunked} diverged from unchunked {unchunked}"
|
||||
)
|
||||
|
||||
@@ -3437,6 +3437,7 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if raw_pooler_output is None or not any(finished_mask):
|
||||
self._sync_device()
|
||||
model_runner_output.pooler_output = [None] * num_reqs
|
||||
return model_runner_output
|
||||
|
||||
|
||||
Reference in New Issue
Block a user