From 39a7f4f4e2635012ead0ad127970d7b6778890af Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:11:04 -0400 Subject: [PATCH] [Perf] Optimize `AllPool.forward` by slicing first, 51% faster in the method level benchmark (#41163) Signed-off-by: yewentao256 --- .../layers/pooler/tokwise/methods.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/vllm/model_executor/layers/pooler/tokwise/methods.py b/vllm/model_executor/layers/pooler/tokwise/methods.py index 523e8e4c9a..d3fefb745c 100644 --- a/vllm/model_executor/layers/pooler/tokwise/methods.py +++ b/vllm/model_executor/layers/pooler/tokwise/methods.py @@ -47,13 +47,17 @@ class AllPool(TokenPoolingMethod): pooling_metadata: PoolingMetadata, ) -> list[TokenPoolingMethodOutputItem]: pooling_cursor = pooling_metadata.get_pooling_cursor() - hidden_states_lst = [ - hidden_states[first : last + 1] - for first, last in zip( - pooling_cursor.first_token_indices_gpu.tolist(), - pooling_cursor.last_token_indices_gpu.tolist(), - ) - ] + split_sizes = pooling_cursor.num_scheduled_tokens_cpu.tolist() + if split_sizes: + # DispatchPooler passes the full hidden_states tensor. + # slice out the subgroup once, then split it by + # per-request token counts + group_start = int(pooling_cursor.first_token_indices_gpu[0].item()) + group_end = int(pooling_cursor.last_token_indices_gpu[-1].item()) + 1 + hidden_states_group = hidden_states[group_start:group_end] + hidden_states_lst = list(hidden_states_group.split(split_sizes)) + else: + hidden_states_lst = [] if not self.enable_chunked_prefill: return hidden_states_lst