[Model Runner V2] Use packed mask for prompt bin counts (#29756)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2025-11-30 14:15:42 -08:00
committed by GitHub
parent 21c2627934
commit ec38a7368d
3 changed files with 35 additions and 25 deletions
+4 -4
View File
@@ -26,7 +26,7 @@ class SamplingMetadata:
# For penalties
idx_mapping: torch.Tensor
prompt_bin_counts: torch.Tensor
prompt_bin_mask: torch.Tensor
output_bin_counts: torch.Tensor
@classmethod
@@ -57,7 +57,7 @@ class SamplingMetadata:
# NOTE(woosuk): These are placeholder tensors to avoid None checks in the
# penalties kernel. We use 2 instead of 1 as vocab_size to avoid Triton
# specialization and re-compilation at runtime.
prompt_bin_counts = torch.zeros(num_reqs, 2, dtype=torch.int32, device=device)
prompt_bin_mask = torch.zeros(num_reqs, 2, dtype=torch.int32, device=device)
output_bin_counts = torch.zeros(num_reqs, 2, dtype=torch.int32, device=device)
return cls(
@@ -71,7 +71,7 @@ class SamplingMetadata:
pos=pos,
max_num_logprobs=max_num_logprobs,
idx_mapping=idx_mapping,
prompt_bin_counts=prompt_bin_counts,
prompt_bin_mask=prompt_bin_mask,
output_bin_counts=output_bin_counts,
)
@@ -174,6 +174,6 @@ def expand_sampling_metadata(
max_num_logprobs=sampling_metadata.max_num_logprobs,
# TODO(woosuk): Support penalties with spec decoding.
idx_mapping=sampling_metadata.idx_mapping,
prompt_bin_counts=sampling_metadata.prompt_bin_counts,
prompt_bin_mask=sampling_metadata.prompt_bin_mask,
output_bin_counts=sampling_metadata.output_bin_counts,
)