[ModelRunner v2] Enable decoder token-wise pooling (#50931)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -40,9 +40,15 @@ class AllPool(TokenPoolingMethod):
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
|
||||
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
|
||||
# With async scheduling, the next step may overwrite the hidden states
|
||||
# buffer before outputs are copied to CPU, so they must own their storage.
|
||||
self.clone_finished = bool(scheduler_config.async_scheduling)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"enable_chunked_prefill={self.enable_chunked_prefill}"
|
||||
return (
|
||||
f"enable_chunked_prefill={self.enable_chunked_prefill}, "
|
||||
f"clone_finished={self.clone_finished}"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -58,18 +64,29 @@ class AllPool(TokenPoolingMethod):
|
||||
)
|
||||
|
||||
if not self.enable_chunked_prefill:
|
||||
if self.clone_finished:
|
||||
return [hs.clone() for hs in hidden_states_lst]
|
||||
return hidden_states_lst
|
||||
|
||||
pooling_states = pooling_metadata.pooling_states
|
||||
finished_mask = pooling_cursor.is_finished().tolist()
|
||||
|
||||
# If chunked_prefill is enabled
|
||||
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
|
||||
for p, hs_chunk in zip(pooling_states, hidden_states_lst):
|
||||
p.hidden_states_cache.append(hs_chunk)
|
||||
for p, hs_chunk, finished in zip(
|
||||
pooling_states, hidden_states_lst, finished_mask
|
||||
):
|
||||
# Own data retained across steps or returned without concatenation.
|
||||
needs_owned_storage = not finished or (
|
||||
self.clone_finished and not p.hidden_states_cache
|
||||
)
|
||||
p.hidden_states_cache.append(
|
||||
hs_chunk.clone() if needs_owned_storage else hs_chunk
|
||||
)
|
||||
|
||||
# 2. Once prefill is finished, send hidden_states_cache to PoolerHead
|
||||
output_list = list[TokenPoolingMethodOutputItem]()
|
||||
for p, finished in zip(pooling_states, pooling_cursor.is_finished()):
|
||||
for p, finished in zip(pooling_states, finished_mask):
|
||||
if finished:
|
||||
hidden_states_cache = p.hidden_states_cache
|
||||
if len(hidden_states_cache) == 1:
|
||||
|
||||
Reference in New Issue
Block a user