[Perf] Remove redundant device copies for CPU-only pooling token IDs, 48.9% E2E throughput improvement (#38139)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-03-29 18:12:50 +00:00
committed by GitHub
parent 8c0b6267d7
commit 995dea1354
8 changed files with 86 additions and 17 deletions
+12 -3
View File
@@ -833,8 +833,13 @@ class InputBatch:
# step pooling during the sampling/pooling process.
# Hence copy these tensors only when there are requests which
# need penalties/step_pooler to be applied.
prompt_token_ids_cpu = (
self._make_prompt_token_ids_cpu_tensor() if needs_prompt_token_ids else None
)
prompt_token_ids = (
self._make_prompt_token_ids_tensor() if needs_prompt_token_ids else None
prompt_token_ids_cpu.to(device=self.device, non_blocking=True)
if prompt_token_ids_cpu is not None
else None
)
# Only set output_token_ids if required by the current requests'
@@ -891,15 +896,19 @@ class InputBatch:
def get_pooling_metadata(self) -> PoolingMetadata:
pooling_params = self.get_pooling_params()
pooling_states = self.get_pooling_states()
prompt_token_ids_cpu = None
if any(p.requires_token_ids for p in pooling_params):
prompt_token_ids_cpu = self._make_prompt_token_ids_cpu_tensor()
return PoolingMetadata(
prompt_lens=self.num_prompt_tokens_cpu_tensor[: self.num_reqs].clone(),
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
prompt_token_ids_cpu=prompt_token_ids_cpu,
pooling_params=pooling_params,
pooling_states=pooling_states,
)
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
def _make_prompt_token_ids_cpu_tensor(self) -> torch.Tensor:
num_reqs = self.num_reqs
max_prompt_len = self.num_prompt_tokens[:num_reqs].max()
prompt_token_ids_cpu_tensor = torch.empty(
@@ -914,7 +923,7 @@ class InputBatch:
# token_id of this value.
for i in range(num_reqs):
prompt_token_ids[i, self.num_prompt_tokens[i] :] = self.vocab_size
return prompt_token_ids_cpu_tensor.to(device=self.device, non_blocking=True)
return prompt_token_ids_cpu_tensor
def make_lora_inputs(
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray