fix bugs when token_classify & classify run concurrently (#36614)
Signed-off-by: augusto.yjh <augusto.yjh@antgroup.com>
This commit is contained in:
@@ -47,10 +47,13 @@ class AllPool(TokenPoolingMethod):
|
|||||||
pooling_metadata: PoolingMetadata,
|
pooling_metadata: PoolingMetadata,
|
||||||
) -> list[TokenPoolingMethodOutputItem]:
|
) -> list[TokenPoolingMethodOutputItem]:
|
||||||
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
||||||
hidden_states_all = hidden_states.split(
|
hidden_states_lst = [
|
||||||
pooling_cursor.num_scheduled_tokens_cpu.tolist()
|
hidden_states[first : last + 1]
|
||||||
)
|
for first, last in zip(
|
||||||
hidden_states_lst = [hidden_states_all[i] for i in pooling_cursor.index]
|
pooling_cursor.first_token_indices_gpu.tolist(),
|
||||||
|
pooling_cursor.last_token_indices_gpu.tolist(),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
if not self.enable_chunked_prefill:
|
if not self.enable_chunked_prefill:
|
||||||
return hidden_states_lst
|
return hidden_states_lst
|
||||||
|
|||||||
Reference in New Issue
Block a user