[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: vx120 <893600387@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
vx120
parent
d6bce42983
commit
560ef78bfe
@@ -2709,6 +2709,13 @@ class VllmConfig:
|
||||
# fixed-width logprobs gather cannot reasonably size for.
|
||||
blockers.append("max_logprobs is -1, allowing vocab-size logprob requests")
|
||||
|
||||
if self.model_config is not None and self.model_config.return_sampling_mask:
|
||||
# gather_sampler_output() drops SamplingMaskTensors: masks come back None.
|
||||
blockers.append(
|
||||
"return_sampling_mask is set and the batch-sharded gather does "
|
||||
"not forward sampling masks"
|
||||
)
|
||||
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.enable_adaptive_verification
|
||||
|
||||
@@ -419,8 +419,9 @@ class RequestState:
|
||||
|
||||
sampling_mask = None
|
||||
if finished and self.sampling_mask_chunks:
|
||||
merged = SamplingMaskLists.merge(self.sampling_mask_chunks)
|
||||
sampling_mask = SamplingMask(merged.to_nested_list())
|
||||
sampling_mask = SamplingMask(
|
||||
[chunk.token_ids.tolist() for chunk in self.sampling_mask_chunks]
|
||||
)
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
routed_experts = None
|
||||
|
||||
+12
-27
@@ -54,42 +54,27 @@ class LogprobsLists(NamedTuple):
|
||||
|
||||
|
||||
class SamplingMaskLists(NamedTuple):
|
||||
"""CSR sampling masks; a step slice holds one position (``offsets=None``)."""
|
||||
|
||||
# [num_kept_tokens]
|
||||
token_ids: np.ndarray
|
||||
# [num_generated_tokens + 1]
|
||||
offsets: np.ndarray
|
||||
# [num_reqs + 1]
|
||||
# [num_positions + 1], or None for a single position
|
||||
offsets: np.ndarray | None = None
|
||||
# Unused with one position per request; kept for the wire layout.
|
||||
cu_num_generated_tokens: list[int] | None = None
|
||||
|
||||
def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists":
|
||||
if self.cu_num_generated_tokens is None:
|
||||
start_idx = req_idx
|
||||
else:
|
||||
start_idx = self.cu_num_generated_tokens[req_idx]
|
||||
end_idx = start_idx + num_positions
|
||||
flat_start = self.offsets[start_idx]
|
||||
flat_end = self.offsets[end_idx]
|
||||
assert num_positions == 1 and self.offsets is not None
|
||||
return SamplingMaskLists(
|
||||
self.token_ids[flat_start:flat_end],
|
||||
self.offsets[start_idx : end_idx + 1] - flat_start,
|
||||
None,
|
||||
self.token_ids[self.offsets[req_idx] : self.offsets[req_idx + 1]]
|
||||
)
|
||||
|
||||
def to_nested_list(self) -> list[list[int]]:
|
||||
"""Convert CSR representation to ``list[list[int]]``."""
|
||||
return [
|
||||
self.token_ids[int(self.offsets[i]) : int(self.offsets[i + 1])].tolist()
|
||||
for i in range(len(self.offsets) - 1)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def merge(chunks: Sequence["SamplingMaskLists"]) -> "SamplingMaskLists":
|
||||
token_ids = np.concatenate([chunk.token_ids for chunk in chunks])
|
||||
counts = np.concatenate([np.diff(chunk.offsets) for chunk in chunks])
|
||||
offsets = np.empty(len(counts) + 1, dtype=np.int64)
|
||||
offsets[0] = 0
|
||||
np.cumsum(counts, dtype=np.int64, out=offsets[1:])
|
||||
return SamplingMaskLists(token_ids, offsets)
|
||||
token_ids = self.token_ids.tolist()
|
||||
if self.offsets is None:
|
||||
return [token_ids]
|
||||
offsets = self.offsets.tolist()
|
||||
return [token_ids[offsets[i] : offsets[i + 1]] for i in range(len(offsets) - 1)]
|
||||
|
||||
|
||||
class LogprobsTensors(NamedTuple):
|
||||
|
||||
@@ -179,7 +179,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
|
||||
if self.sampling_mask_tensors is not None:
|
||||
self.model_runner_output.sampling_masks = (
|
||||
self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np)
|
||||
self.sampling_mask_tensors.tolists()
|
||||
)
|
||||
|
||||
if self.num_nans is not None:
|
||||
|
||||
@@ -23,48 +23,63 @@ class SamplerOutput:
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_sampling_mask_kernel(
|
||||
def _compact_sampling_mask_kernel(
|
||||
logits_ptr,
|
||||
logits_row_stride,
|
||||
logits_col_stride,
|
||||
num_sampled_tokens_ptr,
|
||||
token_ids_ptr,
|
||||
token_ids_row_stride,
|
||||
packed_mask_ptr,
|
||||
packed_mask_row_stride,
|
||||
counts_ptr,
|
||||
vocab_size,
|
||||
max_num_kept,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask."""
|
||||
req_idx = tl.program_id(0)
|
||||
is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
|
||||
for start_idx in range(0, vocab_size, BLOCK_SIZE):
|
||||
offsets = start_idx + tl.arange(0, BLOCK_SIZE)
|
||||
valid = offsets < vocab_size
|
||||
logits = tl.load(
|
||||
logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride,
|
||||
mask=valid,
|
||||
mask=offsets < vocab_size,
|
||||
other=-float("inf"),
|
||||
)
|
||||
keep = (logits > -float("inf")) & (logits < float("inf")) & is_active
|
||||
count += tl.sum(keep).to(tl.int32)
|
||||
keep_i32 = keep.to(tl.int32)
|
||||
pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32
|
||||
tl.store(
|
||||
token_ids_ptr + req_idx * token_ids_row_stride + pos,
|
||||
offsets.to(tl.int32),
|
||||
mask=keep & (pos < max_num_kept),
|
||||
)
|
||||
count += tl.sum(keep_i32)
|
||||
|
||||
keep = tl.reshape(keep.to(tl.int32), (BLOCK_SIZE // 8, 8))
|
||||
bit_shifts = tl.arange(0, 8)[None, :]
|
||||
packed = tl.sum(keep << bit_shifts, axis=1).to(tl.uint8)
|
||||
bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :]
|
||||
byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8)
|
||||
tl.store(
|
||||
packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets,
|
||||
packed,
|
||||
tl.sum(bits, axis=1).to(tl.uint8),
|
||||
mask=byte_offsets < tl.cdiv(vocab_size, 8),
|
||||
)
|
||||
|
||||
tl.store(counts_ptr + req_idx, count)
|
||||
|
||||
|
||||
class SamplingMaskTensors(NamedTuple):
|
||||
"""Bit-packed device-side sampling mask data pending async D2H."""
|
||||
# Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead.
|
||||
MAX_COMPACT_SUPPORT = 2048
|
||||
|
||||
|
||||
class SamplingMaskTensors(NamedTuple):
|
||||
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
|
||||
the exact fallback for rows wider than ``max_num_kept``."""
|
||||
|
||||
# [num_requests, max_num_kept]
|
||||
token_ids: torch.Tensor
|
||||
# [num_requests, ceil(vocab_size / 8)]
|
||||
packed_mask: torch.Tensor
|
||||
# [num_requests]
|
||||
@@ -76,56 +91,63 @@ class SamplingMaskTensors(NamedTuple):
|
||||
cls,
|
||||
logits: torch.Tensor,
|
||||
num_sampled_tokens: torch.Tensor,
|
||||
max_num_kept: int,
|
||||
) -> SamplingMaskTensors:
|
||||
"""Pack the finite-logit support for requests that sampled tokens."""
|
||||
"""Capture the finite-logit support of every row with a sampled token."""
|
||||
num_reqs, vocab_size = logits.shape
|
||||
packed_width = (vocab_size + 7) // 8
|
||||
max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT)
|
||||
device = logits.device
|
||||
|
||||
packed_mask = torch.empty(
|
||||
(num_reqs, packed_width), dtype=torch.uint8, device=logits.device
|
||||
token_ids = torch.empty(
|
||||
(num_reqs, max_num_kept), dtype=torch.int32, device=device
|
||||
)
|
||||
counts = torch.empty(num_reqs, dtype=torch.int32, device=logits.device)
|
||||
_pack_sampling_mask_kernel[(num_reqs,)](
|
||||
packed_mask = torch.empty(
|
||||
(num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device
|
||||
)
|
||||
counts = torch.empty(num_reqs, dtype=torch.int32, device=device)
|
||||
_compact_sampling_mask_kernel[(num_reqs,)](
|
||||
logits,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
num_sampled_tokens,
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
packed_mask,
|
||||
packed_mask.stride(0),
|
||||
counts,
|
||||
vocab_size,
|
||||
max_num_kept,
|
||||
BLOCK_SIZE=8192,
|
||||
)
|
||||
|
||||
return cls(packed_mask, counts, vocab_size)
|
||||
return cls(token_ids, packed_mask, counts, vocab_size)
|
||||
|
||||
def to_cpu_nonblocking(self) -> SamplingMaskTensors:
|
||||
if self.packed_mask.device.type == "cpu":
|
||||
if self.token_ids.device.type == "cpu":
|
||||
return self
|
||||
return SamplingMaskTensors(
|
||||
self.token_ids.to("cpu", non_blocking=True),
|
||||
self.packed_mask.to("cpu", non_blocking=True),
|
||||
self.counts.to("cpu", non_blocking=True),
|
||||
self.vocab_size,
|
||||
)
|
||||
|
||||
def tolists(self, num_sampled_tokens: np.ndarray) -> SamplingMaskLists:
|
||||
"""Convert the packed masks to the scheduler's CSR representation."""
|
||||
sampled_rows = np.flatnonzero(num_sampled_tokens)
|
||||
counts = self.counts.cpu().numpy()[sampled_rows]
|
||||
offsets = np.empty(len(counts) + 1, dtype=np.int64)
|
||||
offsets[0] = 0
|
||||
np.cumsum(counts, dtype=np.int64, out=offsets[1:])
|
||||
unpacked = np.unpackbits(
|
||||
self.packed_mask.cpu().numpy()[sampled_rows],
|
||||
axis=1,
|
||||
count=self.vocab_size,
|
||||
bitorder="little",
|
||||
)
|
||||
token_ids = np.nonzero(unpacked)[1].astype(np.int32, copy=False)
|
||||
return SamplingMaskLists(
|
||||
token_ids=token_ids,
|
||||
offsets=offsets,
|
||||
cu_num_generated_tokens=np.cumsum(
|
||||
np.concatenate(([0], num_sampled_tokens))
|
||||
).tolist(),
|
||||
)
|
||||
def tolists(self) -> SamplingMaskLists:
|
||||
"""CSR over all requests; rows without a sampled token are empty."""
|
||||
counts = self.counts.cpu().numpy()
|
||||
token_ids = self.token_ids.cpu().numpy()
|
||||
packed_mask = self.packed_mask.cpu().numpy()
|
||||
width = token_ids.shape[1]
|
||||
|
||||
def support(row: int) -> np.ndarray:
|
||||
if counts[row] <= width:
|
||||
return token_ids[row, : counts[row]]
|
||||
# Wider than the compact row (ties or a huge top_k): use the bitmask.
|
||||
bits = np.unpackbits(
|
||||
packed_mask[row], count=self.vocab_size, bitorder="little"
|
||||
)
|
||||
return np.flatnonzero(bits).astype(np.int32, copy=False)
|
||||
|
||||
supports = [support(row) for row in range(len(counts))]
|
||||
offsets = np.zeros(len(supports) + 1, dtype=np.int64)
|
||||
np.cumsum([len(s) for s in supports], out=offsets[1:])
|
||||
return SamplingMaskLists(np.concatenate(supports), offsets)
|
||||
|
||||
@@ -186,8 +186,10 @@ class Sampler:
|
||||
|
||||
sampling_mask_tensors = None
|
||||
if self.return_sampling_mask:
|
||||
# Size by the validated top_k batch max; wider supports use the bitmask.
|
||||
max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np]))
|
||||
sampling_mask_tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits, num_sampled
|
||||
processed_logits, num_sampled, max_num_kept
|
||||
)
|
||||
|
||||
# These are GPU tensors.
|
||||
|
||||
Reference in New Issue
Block a user