[Perf] Remove redundant device copies for CPU-only pooling token IDs, 48.9% E2E throughput improvement (#38139)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -833,8 +833,13 @@ class InputBatch:
|
||||
# step pooling during the sampling/pooling process.
|
||||
# Hence copy these tensors only when there are requests which
|
||||
# need penalties/step_pooler to be applied.
|
||||
prompt_token_ids_cpu = (
|
||||
self._make_prompt_token_ids_cpu_tensor() if needs_prompt_token_ids else None
|
||||
)
|
||||
prompt_token_ids = (
|
||||
self._make_prompt_token_ids_tensor() if needs_prompt_token_ids else None
|
||||
prompt_token_ids_cpu.to(device=self.device, non_blocking=True)
|
||||
if prompt_token_ids_cpu is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Only set output_token_ids if required by the current requests'
|
||||
@@ -891,15 +896,19 @@ class InputBatch:
|
||||
def get_pooling_metadata(self) -> PoolingMetadata:
|
||||
pooling_params = self.get_pooling_params()
|
||||
pooling_states = self.get_pooling_states()
|
||||
prompt_token_ids_cpu = None
|
||||
if any(p.requires_token_ids for p in pooling_params):
|
||||
prompt_token_ids_cpu = self._make_prompt_token_ids_cpu_tensor()
|
||||
|
||||
return PoolingMetadata(
|
||||
prompt_lens=self.num_prompt_tokens_cpu_tensor[: self.num_reqs].clone(),
|
||||
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=pooling_states,
|
||||
)
|
||||
|
||||
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
|
||||
def _make_prompt_token_ids_cpu_tensor(self) -> torch.Tensor:
|
||||
num_reqs = self.num_reqs
|
||||
max_prompt_len = self.num_prompt_tokens[:num_reqs].max()
|
||||
prompt_token_ids_cpu_tensor = torch.empty(
|
||||
@@ -914,7 +923,7 @@ class InputBatch:
|
||||
# token_id of this value.
|
||||
for i in range(num_reqs):
|
||||
prompt_token_ids[i, self.num_prompt_tokens[i] :] = self.vocab_size
|
||||
return prompt_token_ids_cpu_tensor.to(device=self.device, non_blocking=True)
|
||||
return prompt_token_ids_cpu_tensor
|
||||
|
||||
def make_lora_inputs(
|
||||
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray
|
||||
|
||||
Reference in New Issue
Block a user