[Bugfix] Prevent padding placeholders from reaching embeddings (#47029)
Signed-off-by: qianlihuang <91178480+qianlihuang@users.noreply.github.com> Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -435,6 +435,40 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
)
|
||||
|
||||
|
||||
def test_placeholder_spec_token_ids_written_verbatim():
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=1,
|
||||
max_model_len=8,
|
||||
max_num_batched_tokens=8,
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
req = CachedRequestState(
|
||||
req_id="req",
|
||||
prompt_token_ids=[10, 11],
|
||||
mm_features=[],
|
||||
sampling_params=SamplingParams(),
|
||||
block_ids=([],),
|
||||
generator=None,
|
||||
num_computed_tokens=3,
|
||||
output_token_ids=[12],
|
||||
)
|
||||
input_batch.add_request(req)
|
||||
|
||||
input_batch.update_req_spec_token_ids(
|
||||
req,
|
||||
{"req": [13, -1, -1]},
|
||||
)
|
||||
|
||||
# Placeholders (-1) are kept verbatim in both the spec_token_ids list and
|
||||
# the token buffer; they are clamped to 0 only at the embedding boundary
|
||||
# (GPUModelRunner._preprocess).
|
||||
assert input_batch.spec_token_ids[0] == [13, -1, -1]
|
||||
assert input_batch.token_ids_cpu[0, 3:6].tolist() == [13, -1, -1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pooling_params", "expect_device_prompt_token_ids", "expect_cpu_prompt_token_ids"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user