[Bugfix] Prevent padding placeholders from reaching embeddings (#47029)

Signed-off-by: qianlihuang <91178480+qianlihuang@users.noreply.github.com>
Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Yiliu Dong
2026-07-01 09:26:03 -07:00
committed by GitHub
co-authored by Nick Hill
parent c8bdcc0116
commit 00eb7cefa3
3 changed files with 60 additions and 0 deletions
+34
View File
@@ -435,6 +435,40 @@ def test_pooling_prompt_lens_not_aliased(device: str):
)
def test_placeholder_spec_token_ids_written_verbatim():
input_batch = InputBatch(
max_num_reqs=1,
max_model_len=8,
max_num_batched_tokens=8,
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
)
req = CachedRequestState(
req_id="req",
prompt_token_ids=[10, 11],
mm_features=[],
sampling_params=SamplingParams(),
block_ids=([],),
generator=None,
num_computed_tokens=3,
output_token_ids=[12],
)
input_batch.add_request(req)
input_batch.update_req_spec_token_ids(
req,
{"req": [13, -1, -1]},
)
# Placeholders (-1) are kept verbatim in both the spec_token_ids list and
# the token buffer; they are clamped to 0 only at the embedding boundary
# (GPUModelRunner._preprocess).
assert input_batch.spec_token_ids[0] == [13, -1, -1]
assert input_batch.token_ids_cpu[0, 3:6].tolist() == [13, -1, -1]
@pytest.mark.parametrize(
("pooling_params", "expect_device_prompt_token_ids", "expect_cpu_prompt_token_ids"),
[