[perf][async] support non cpu sync get logprob tensors for spec (#31336)
Signed-off-by: izhuhaoran <izhuhaoran@qq.com> Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -237,19 +237,20 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist()
|
||||
for i in self._invalid_req_indices:
|
||||
valid_sampled_token_ids[i].clear()
|
||||
cu_num_tokens = None
|
||||
logprobs_lists = None
|
||||
if self._logprobs_tensors_cpu is not None:
|
||||
logprobs_lists = self._logprobs_tensors_cpu.tolists()
|
||||
else:
|
||||
valid_sampled_token_ids, cu_num_tokens = RejectionSampler.parse_output(
|
||||
valid_sampled_token_ids, logprobs_lists = RejectionSampler.parse_output(
|
||||
self.sampled_token_ids_cpu,
|
||||
self.vocab_size,
|
||||
self._invalid_req_indices,
|
||||
return_cu_num_tokens=self._logprobs_tensors_cpu is not None,
|
||||
logprobs_tensors=self._logprobs_tensors_cpu,
|
||||
)
|
||||
|
||||
output = self._model_runner_output
|
||||
output.sampled_token_ids = valid_sampled_token_ids
|
||||
if self._logprobs_tensors_cpu:
|
||||
output.logprobs = self._logprobs_tensors_cpu.tolists(cu_num_tokens)
|
||||
output.logprobs = logprobs_lists
|
||||
return output
|
||||
|
||||
|
||||
@@ -395,6 +396,9 @@ class GPUModelRunner(
|
||||
else:
|
||||
self.max_encoder_len = 0
|
||||
|
||||
# Async scheduling
|
||||
self.use_async_scheduling = self.scheduler_config.async_scheduling
|
||||
|
||||
# Sampler
|
||||
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
|
||||
|
||||
@@ -504,7 +508,6 @@ class GPUModelRunner(
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
self.use_async_scheduling = self.scheduler_config.async_scheduling
|
||||
# Separate cuda stream for overlapping transfer of sampled token ids from
|
||||
# GPU to CPU when async scheduling is enabled.
|
||||
self.async_output_copy_stream: torch.cuda.Stream | None = None
|
||||
@@ -2784,7 +2787,7 @@ class GPUModelRunner(
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
logprobs_tensors = sampler_output.logprobs_tensors
|
||||
invalid_req_indices = []
|
||||
cu_num_tokens: list[int] | None = None
|
||||
logprobs_lists = None
|
||||
if not self.use_async_scheduling:
|
||||
# Get the valid generated tokens.
|
||||
max_gen_len = sampled_token_ids.shape[-1]
|
||||
@@ -2794,13 +2797,16 @@ class GPUModelRunner(
|
||||
# Mask out the sampled tokens that should not be sampled.
|
||||
for i in discard_sampled_tokens_req_indices:
|
||||
valid_sampled_token_ids[int(i)].clear()
|
||||
|
||||
if logprobs_tensors is not None:
|
||||
logprobs_lists = logprobs_tensors.tolists()
|
||||
else:
|
||||
# Includes spec decode tokens.
|
||||
valid_sampled_token_ids, cu_num_tokens = RejectionSampler.parse_output(
|
||||
valid_sampled_token_ids, logprobs_lists = RejectionSampler.parse_output(
|
||||
sampled_token_ids,
|
||||
self.input_batch.vocab_size,
|
||||
discard_sampled_tokens_req_indices,
|
||||
return_cu_num_tokens=logprobs_tensors is not None,
|
||||
logprobs_tensors=logprobs_tensors,
|
||||
)
|
||||
else:
|
||||
valid_sampled_token_ids = []
|
||||
@@ -2853,12 +2859,6 @@ class GPUModelRunner(
|
||||
req_state = self.requests[req_id]
|
||||
req_state.output_token_ids.extend(sampled_ids)
|
||||
|
||||
logprobs_lists = (
|
||||
logprobs_tensors.tolists(cu_num_tokens)
|
||||
if not self.use_async_scheduling and logprobs_tensors is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Compute prompt logprobs if needed.
|
||||
prompt_logprobs_dict = self._get_prompt_logprobs_dict(
|
||||
hidden_states[:num_scheduled_tokens],
|
||||
|
||||
Reference in New Issue
Block a user