[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: vx120 <893600387@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
aoshen02
2026-09-03 19:03:36 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 vx120
parent d6bce42983
commit 560ef78bfe
10 changed files with 188 additions and 108 deletions
+16
View File
@@ -753,6 +753,22 @@ def test_v1_model_runner_rejects_v2_only_features():
VllmConfig._validate_v1_model_runner(config)
def test_batch_sharded_sampling_rejects_return_sampling_mask():
"""The batch-sharded gather drops sampling masks, so the combination must
fail loudly instead of returning ``sampling_mask=None``."""
config = SimpleNamespace(
parallel_config=SimpleNamespace(
enable_batch_sharded_sampling=True, tensor_parallel_size=2
),
scheduler_config=SimpleNamespace(max_num_seqs=8),
model_config=SimpleNamespace(max_logprobs=20, return_sampling_mask=True),
speculative_config=None,
)
with pytest.raises(ValueError, match="sampling masks"):
VllmConfig._validate_batch_sharded_sampling(config)
@pytest.mark.skip_global_cleanup
def test_with_hf_config_populates_missing_architectures_from_causal_lm_mapping(
monkeypatch,