[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: vx120 <893600387@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
aoshen02
2026-09-03 19:03:36 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 vx120
parent d6bce42983
commit 560ef78bfe
10 changed files with 188 additions and 108 deletions
+62 -40
View File
@@ -23,48 +23,63 @@ class SamplerOutput:
@triton.jit
def _pack_sampling_mask_kernel(
def _compact_sampling_mask_kernel(
logits_ptr,
logits_row_stride,
logits_col_stride,
num_sampled_tokens_ptr,
token_ids_ptr,
token_ids_row_stride,
packed_mask_ptr,
packed_mask_row_stride,
counts_ptr,
vocab_size,
max_num_kept,
BLOCK_SIZE: tl.constexpr,
):
"""Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask."""
req_idx = tl.program_id(0)
is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0
count = tl.zeros((), dtype=tl.int32)
for start_idx in range(0, vocab_size, BLOCK_SIZE):
offsets = start_idx + tl.arange(0, BLOCK_SIZE)
valid = offsets < vocab_size
logits = tl.load(
logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride,
mask=valid,
mask=offsets < vocab_size,
other=-float("inf"),
)
keep = (logits > -float("inf")) & (logits < float("inf")) & is_active
count += tl.sum(keep).to(tl.int32)
keep_i32 = keep.to(tl.int32)
pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32
tl.store(
token_ids_ptr + req_idx * token_ids_row_stride + pos,
offsets.to(tl.int32),
mask=keep & (pos < max_num_kept),
)
count += tl.sum(keep_i32)
keep = tl.reshape(keep.to(tl.int32), (BLOCK_SIZE // 8, 8))
bit_shifts = tl.arange(0, 8)[None, :]
packed = tl.sum(keep << bit_shifts, axis=1).to(tl.uint8)
bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :]
byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8)
tl.store(
packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets,
packed,
tl.sum(bits, axis=1).to(tl.uint8),
mask=byte_offsets < tl.cdiv(vocab_size, 8),
)
tl.store(counts_ptr + req_idx, count)
class SamplingMaskTensors(NamedTuple):
"""Bit-packed device-side sampling mask data pending async D2H."""
# Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead.
MAX_COMPACT_SUPPORT = 2048
class SamplingMaskTensors(NamedTuple):
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
the exact fallback for rows wider than ``max_num_kept``."""
# [num_requests, max_num_kept]
token_ids: torch.Tensor
# [num_requests, ceil(vocab_size / 8)]
packed_mask: torch.Tensor
# [num_requests]
@@ -76,56 +91,63 @@ class SamplingMaskTensors(NamedTuple):
cls,
logits: torch.Tensor,
num_sampled_tokens: torch.Tensor,
max_num_kept: int,
) -> SamplingMaskTensors:
"""Pack the finite-logit support for requests that sampled tokens."""
"""Capture the finite-logit support of every row with a sampled token."""
num_reqs, vocab_size = logits.shape
packed_width = (vocab_size + 7) // 8
max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT)
device = logits.device
packed_mask = torch.empty(
(num_reqs, packed_width), dtype=torch.uint8, device=logits.device
token_ids = torch.empty(
(num_reqs, max_num_kept), dtype=torch.int32, device=device
)
counts = torch.empty(num_reqs, dtype=torch.int32, device=logits.device)
_pack_sampling_mask_kernel[(num_reqs,)](
packed_mask = torch.empty(
(num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device
)
counts = torch.empty(num_reqs, dtype=torch.int32, device=device)
_compact_sampling_mask_kernel[(num_reqs,)](
logits,
logits.stride(0),
logits.stride(1),
num_sampled_tokens,
token_ids,
token_ids.stride(0),
packed_mask,
packed_mask.stride(0),
counts,
vocab_size,
max_num_kept,
BLOCK_SIZE=8192,
)
return cls(packed_mask, counts, vocab_size)
return cls(token_ids, packed_mask, counts, vocab_size)
def to_cpu_nonblocking(self) -> SamplingMaskTensors:
if self.packed_mask.device.type == "cpu":
if self.token_ids.device.type == "cpu":
return self
return SamplingMaskTensors(
self.token_ids.to("cpu", non_blocking=True),
self.packed_mask.to("cpu", non_blocking=True),
self.counts.to("cpu", non_blocking=True),
self.vocab_size,
)
def tolists(self, num_sampled_tokens: np.ndarray) -> SamplingMaskLists:
"""Convert the packed masks to the scheduler's CSR representation."""
sampled_rows = np.flatnonzero(num_sampled_tokens)
counts = self.counts.cpu().numpy()[sampled_rows]
offsets = np.empty(len(counts) + 1, dtype=np.int64)
offsets[0] = 0
np.cumsum(counts, dtype=np.int64, out=offsets[1:])
unpacked = np.unpackbits(
self.packed_mask.cpu().numpy()[sampled_rows],
axis=1,
count=self.vocab_size,
bitorder="little",
)
token_ids = np.nonzero(unpacked)[1].astype(np.int32, copy=False)
return SamplingMaskLists(
token_ids=token_ids,
offsets=offsets,
cu_num_generated_tokens=np.cumsum(
np.concatenate(([0], num_sampled_tokens))
).tolist(),
)
def tolists(self) -> SamplingMaskLists:
"""CSR over all requests; rows without a sampled token are empty."""
counts = self.counts.cpu().numpy()
token_ids = self.token_ids.cpu().numpy()
packed_mask = self.packed_mask.cpu().numpy()
width = token_ids.shape[1]
def support(row: int) -> np.ndarray:
if counts[row] <= width:
return token_ids[row, : counts[row]]
# Wider than the compact row (ties or a huge top_k): use the bitmask.
bits = np.unpackbits(
packed_mask[row], count=self.vocab_size, bitorder="little"
)
return np.flatnonzero(bits).astype(np.int32, copy=False)
supports = [support(row) for row in range(len(counts))]
offsets = np.zeros(len(supports) + 1, dtype=np.int64)
np.cumsum([len(s) for s in supports], out=offsets[1:])
return SamplingMaskLists(np.concatenate(supports), offsets)
+3 -1
View File
@@ -186,8 +186,10 @@ class Sampler:
sampling_mask_tensors = None
if self.return_sampling_mask:
# Size by the validated top_k batch max; wider supports use the bitmask.
max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np]))
sampling_mask_tensors = SamplingMaskTensors.from_logits(
processed_logits, num_sampled
processed_logits, num_sampled, max_num_kept
)
# These are GPU tensors.