[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -18,7 +18,7 @@ ActivationFn = Callable[[_T], _T]
|
||||
@dataclass(frozen=True)
|
||||
class PoolingParamsUpdate:
|
||||
requires_token_ids: bool = False
|
||||
"""Set this flag to enable prompt token IDs for your pooler."""
|
||||
"""Set this flag to enable CPU prompt token IDs for your pooler."""
|
||||
|
||||
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
|
||||
return PoolingParamsUpdate(
|
||||
|
||||
@@ -69,7 +69,7 @@ class AllPool(TokenPoolingMethod):
|
||||
return hidden_states_lst
|
||||
|
||||
pooling_states = pooling_metadata.pooling_states
|
||||
finished_mask = pooling_cursor.is_finished().tolist()
|
||||
finished_mask = pooling_cursor.get_finished_mask()
|
||||
|
||||
# If chunked_prefill is enabled
|
||||
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
|
||||
|
||||
Reference in New Issue
Block a user