[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)
Signed-off-by: Bowen Wang <bowen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Nick Hill
vllm-agent
parent
556ac15eb9
commit
7b6310c109
@@ -1404,14 +1404,17 @@ class DiffusionSampler:
|
||||
# unaffected; only the canvas exploration is constrained. Applied
|
||||
# before canvas padding so phantom positions stay uniform.
|
||||
if num_decode > 0:
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
decode_slots.repeat_interleave(
|
||||
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
|
||||
),
|
||||
decode_slots_np,
|
||||
logits.shape[-1],
|
||||
)
|
||||
if top_k is not None or top_p is not None:
|
||||
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
|
||||
logits = apply_top_k_top_p(
|
||||
logits.float(), top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
# Where each decode request's rows start in the flat logits. Tiles
|
||||
# below gather and pad a request to the tile's width.
|
||||
|
||||
Reference in New Issue
Block a user