[Perf] Remove redundant device copies for CPU-only pooling token IDs, 48.9% E2E throughput improvement (#38139)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -18,7 +18,7 @@ ActivationFn = Callable[[_T], _T]
|
||||
@dataclass(frozen=True)
|
||||
class PoolingParamsUpdate:
|
||||
requires_token_ids: bool = False
|
||||
"""Set this flag to enable `get_prompt_token_ids` for your pooler."""
|
||||
"""Set this flag to enable prompt token IDs for your pooler."""
|
||||
|
||||
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
|
||||
return PoolingParamsUpdate(
|
||||
|
||||
@@ -146,17 +146,19 @@ class BOSEOSFilter(Pooler):
|
||||
) -> PoolerOutput:
|
||||
pooled_outputs = self.pooler(hidden_states, pooling_metadata)
|
||||
assert isinstance(pooled_outputs, list)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
|
||||
for i, prompt_len in enumerate(pooling_metadata.prompt_lens):
|
||||
for i, (prompt_len, token_ids) in enumerate(
|
||||
zip(pooling_metadata.prompt_lens, prompt_token_ids)
|
||||
):
|
||||
pooled_data = pooled_outputs[i]
|
||||
assert (
|
||||
isinstance(pooled_data, torch.Tensor)
|
||||
and pooled_data.shape[0] == prompt_len
|
||||
)
|
||||
token_ids = pooling_metadata.prompt_token_ids[i, :prompt_len]
|
||||
if token_ids[0] == self.bos_token_id:
|
||||
if int(token_ids[0]) == self.bos_token_id:
|
||||
pooled_data = pooled_data[1:]
|
||||
if token_ids[-1] == self.eos_token_id:
|
||||
if int(token_ids[-1]) == self.eos_token_id:
|
||||
pooled_data = pooled_data[:-1]
|
||||
pooled_outputs[i] = pooled_data.squeeze(-1)
|
||||
|
||||
|
||||
@@ -638,25 +638,26 @@ class SPLADESparsePooler(Pooler):
|
||||
lens: list[int] = lens_tensor.tolist()
|
||||
B: int = len(lens)
|
||||
|
||||
token_ids = pooling_metadata.prompt_token_ids
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
offset = 0
|
||||
pooled_list: list[torch.Tensor] = []
|
||||
|
||||
for i in range(B):
|
||||
L = int(lens[i])
|
||||
hs = hidden_states[offset : offset + L]
|
||||
token_ids = prompt_token_ids[i]
|
||||
|
||||
start_idx = 0
|
||||
end_idx = L
|
||||
if self.remove_cls_sep and token_ids is not None:
|
||||
if self.remove_cls_sep:
|
||||
if (
|
||||
self.cls_token_id is not None
|
||||
and token_ids[i, 0].item() == self.cls_token_id
|
||||
and int(token_ids[0]) == self.cls_token_id
|
||||
):
|
||||
start_idx = 1
|
||||
if (
|
||||
self.sep_token_id is not None
|
||||
and token_ids[i, L - 1].item() == self.sep_token_id
|
||||
and int(token_ids[L - 1]) == self.sep_token_id
|
||||
):
|
||||
end_idx = max(start_idx, L - 1)
|
||||
|
||||
|
||||
@@ -156,10 +156,11 @@ class GritLMMeanPool(SequencePoolingMethod):
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
prompt_lens = pooling_metadata.prompt_lens
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
instr_lens = torch.tensor(
|
||||
[
|
||||
self._get_instruction_len(token_ids.cpu().numpy())
|
||||
for token_ids in pooling_metadata.get_prompt_token_ids()
|
||||
self._get_instruction_len(token_ids.numpy())
|
||||
for token_ids in prompt_token_ids
|
||||
],
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user