[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: vx120 <893600387@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
aoshen02
2026-09-03 19:03:36 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 vx120
parent d6bce42983
commit 560ef78bfe
10 changed files with 188 additions and 108 deletions
@@ -137,6 +137,7 @@ async def test_generate_sampling_mask(client):
"top_p": 0.9,
"ignore_eos": True,
"seed": 0,
"logprobs": top_k,
},
"stream": False,
}
@@ -150,11 +151,16 @@ async def test_generate_sampling_mask(client):
assert len(token_ids) == len(sampling_mask)
vocab_size = get_vocab_size(MODEL_NAME)
for token_id, support in zip(token_ids, sampling_mask):
logprobs_content = choice["logprobs"]["content"]
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
assert support
assert len(support) == len(set(support))
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
assert token_id in support
# processed_logprobs: exactly the support carries finite probability.
for top in entry["top_logprobs"]:
in_support = int(top["token"].removeprefix("token_id:")) in support
assert in_support == (top["logprob"] > -9999.0)
@pytest.mark.asyncio