[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: vx120 <893600387@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
vx120
parent
d6bce42983
commit
560ef78bfe
@@ -137,6 +137,7 @@ async def test_generate_sampling_mask(client):
|
||||
"top_p": 0.9,
|
||||
"ignore_eos": True,
|
||||
"seed": 0,
|
||||
"logprobs": top_k,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
@@ -150,11 +151,16 @@ async def test_generate_sampling_mask(client):
|
||||
assert len(token_ids) == len(sampling_mask)
|
||||
|
||||
vocab_size = get_vocab_size(MODEL_NAME)
|
||||
for token_id, support in zip(token_ids, sampling_mask):
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
|
||||
assert support
|
||||
assert len(support) == len(set(support))
|
||||
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
|
||||
assert token_id in support
|
||||
# processed_logprobs: exactly the support carries finite probability.
|
||||
for top in entry["top_logprobs"]:
|
||||
in_support = int(top["token"].removeprefix("token_id:")) in support
|
||||
assert in_support == (top["logprob"] > -9999.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user