[perf][async] support non cpu sync get logprob tensors for spec (#31336)

Signed-off-by: izhuhaoran <izhuhaoran@qq.com>
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
zhrrr
2026-01-09 21:24:51 +00:00
committed by GitHub
parent 94578127a4
commit 97ba96fbe9
3 changed files with 48 additions and 29 deletions
+15 -15
View File
@@ -237,19 +237,20 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist()
for i in self._invalid_req_indices:
valid_sampled_token_ids[i].clear()
cu_num_tokens = None
logprobs_lists = None
if self._logprobs_tensors_cpu is not None:
logprobs_lists = self._logprobs_tensors_cpu.tolists()
else:
valid_sampled_token_ids, cu_num_tokens = RejectionSampler.parse_output(
valid_sampled_token_ids, logprobs_lists = RejectionSampler.parse_output(
self.sampled_token_ids_cpu,
self.vocab_size,
self._invalid_req_indices,
return_cu_num_tokens=self._logprobs_tensors_cpu is not None,
logprobs_tensors=self._logprobs_tensors_cpu,
)
output = self._model_runner_output
output.sampled_token_ids = valid_sampled_token_ids
if self._logprobs_tensors_cpu:
output.logprobs = self._logprobs_tensors_cpu.tolists(cu_num_tokens)
output.logprobs = logprobs_lists
return output
@@ -395,6 +396,9 @@ class GPUModelRunner(
else:
self.max_encoder_len = 0
# Async scheduling
self.use_async_scheduling = self.scheduler_config.async_scheduling
# Sampler
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
@@ -504,7 +508,6 @@ class GPUModelRunner(
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
)
self.use_async_scheduling = self.scheduler_config.async_scheduling
# Separate cuda stream for overlapping transfer of sampled token ids from
# GPU to CPU when async scheduling is enabled.
self.async_output_copy_stream: torch.cuda.Stream | None = None
@@ -2784,7 +2787,7 @@ class GPUModelRunner(
sampled_token_ids = sampler_output.sampled_token_ids
logprobs_tensors = sampler_output.logprobs_tensors
invalid_req_indices = []
cu_num_tokens: list[int] | None = None
logprobs_lists = None
if not self.use_async_scheduling:
# Get the valid generated tokens.
max_gen_len = sampled_token_ids.shape[-1]
@@ -2794,13 +2797,16 @@ class GPUModelRunner(
# Mask out the sampled tokens that should not be sampled.
for i in discard_sampled_tokens_req_indices:
valid_sampled_token_ids[int(i)].clear()
if logprobs_tensors is not None:
logprobs_lists = logprobs_tensors.tolists()
else:
# Includes spec decode tokens.
valid_sampled_token_ids, cu_num_tokens = RejectionSampler.parse_output(
valid_sampled_token_ids, logprobs_lists = RejectionSampler.parse_output(
sampled_token_ids,
self.input_batch.vocab_size,
discard_sampled_tokens_req_indices,
return_cu_num_tokens=logprobs_tensors is not None,
logprobs_tensors=logprobs_tensors,
)
else:
valid_sampled_token_ids = []
@@ -2853,12 +2859,6 @@ class GPUModelRunner(
req_state = self.requests[req_id]
req_state.output_token_ids.extend(sampled_ids)
logprobs_lists = (
logprobs_tensors.tolists(cu_num_tokens)
if not self.use_async_scheduling and logprobs_tensors is not None
else None
)
# Compute prompt logprobs if needed.
prompt_logprobs_dict = self._get_prompt_logprobs_dict(
hidden_states[:num_scheduled_tokens],