[Bugfix][Scheduler] Let pooling chunked prefill use full context (#48039)
Signed-off-by: Tommy <tran.tommy@hotmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Harry Mellor
parent
cbee82324c
commit
e6fc81bc78
@@ -147,9 +147,15 @@ class Scheduler(SchedulerInterface):
|
||||
self.kv_events_config is not None
|
||||
and self.kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
# Diffusion models may not sample any tokens for a denoising step.
|
||||
# Pooling and diffusion models do not append a sampled token after
|
||||
# prefill, so they should not reserve one context slot while scheduling.
|
||||
self.num_sampled_tokens_per_step = (
|
||||
1 if not vllm_config.model_config.is_diffusion else 0
|
||||
0
|
||||
if (
|
||||
vllm_config.model_config.runner_type == "pooling"
|
||||
or vllm_config.model_config.is_diffusion
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
# Create KVConnector for the Scheduler. Note that each Worker
|
||||
|
||||
Reference in New Issue
Block a user