[Bugfix][Scheduler] Let pooling chunked prefill use full context (#48039)
Signed-off-by: Tommy <tran.tommy@hotmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Harry Mellor
parent
cbee82324c
commit
e6fc81bc78
@@ -35,6 +35,7 @@ from vllm.multimodal.inputs import (
|
|||||||
MultiModalKwargsItem,
|
MultiModalKwargsItem,
|
||||||
PlaceholderRange,
|
PlaceholderRange,
|
||||||
)
|
)
|
||||||
|
from vllm.pooling_params import PoolingParams
|
||||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||||
from vllm.utils.hashing import sha256
|
from vllm.utils.hashing import sha256
|
||||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||||
@@ -279,6 +280,24 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
|
|||||||
assert scheduler.running[i] == request
|
assert scheduler.running[i] == request
|
||||||
|
|
||||||
|
|
||||||
|
def test_pooling_chunked_prefill_can_finish_at_max_model_len():
|
||||||
|
scheduler = create_scheduler(
|
||||||
|
max_num_seqs=1, max_model_len=8, max_num_batched_tokens=4, runner="pooling"
|
||||||
|
)
|
||||||
|
assert scheduler.max_model_len == 8
|
||||||
|
request = Request(
|
||||||
|
request_id="pool",
|
||||||
|
prompt_token_ids=[1] * 8,
|
||||||
|
sampling_params=None,
|
||||||
|
pooling_params=PoolingParams(task="embed"),
|
||||||
|
)
|
||||||
|
scheduler.add_request(request)
|
||||||
|
|
||||||
|
scheduler.schedule()
|
||||||
|
# The final chunk must reach max_model_len, not stop one token short.
|
||||||
|
assert scheduler.schedule().num_scheduled_tokens == {"pool": 4}
|
||||||
|
|
||||||
|
|
||||||
def test_scheduler_stats_route_to_existing_output_client():
|
def test_scheduler_stats_route_to_existing_output_client():
|
||||||
scheduler = create_scheduler()
|
scheduler = create_scheduler()
|
||||||
request = create_requests(num_requests=1)[0]
|
request = create_requests(num_requests=1)[0]
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from vllm.config import (
|
|||||||
SpeculativeConfig,
|
SpeculativeConfig,
|
||||||
VllmConfig,
|
VllmConfig,
|
||||||
)
|
)
|
||||||
|
from vllm.config.model import RunnerOption
|
||||||
from vllm.config.scheduler import SchedulerPolicy
|
from vllm.config.scheduler import SchedulerPolicy
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
@@ -78,6 +79,7 @@ def create_scheduler(
|
|||||||
ec_role: str | None = None,
|
ec_role: str | None = None,
|
||||||
use_v2_model_runner: bool | None = None,
|
use_v2_model_runner: bool | None = None,
|
||||||
kv_cache_spec: KVCacheSpec | None = None,
|
kv_cache_spec: KVCacheSpec | None = None,
|
||||||
|
runner: RunnerOption = "auto",
|
||||||
per_request_spec_decode_metrics: str = "none",
|
per_request_spec_decode_metrics: str = "none",
|
||||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||||
diffusion_canvas_length: int | None = None,
|
diffusion_canvas_length: int | None = None,
|
||||||
@@ -109,6 +111,7 @@ def create_scheduler(
|
|||||||
# The scheduler reads model_config.max_model_len, not the
|
# The scheduler reads model_config.max_model_len, not the
|
||||||
# SchedulerConfig one, so both must agree.
|
# SchedulerConfig one, so both must agree.
|
||||||
max_model_len=max_model_len,
|
max_model_len=max_model_len,
|
||||||
|
runner=runner,
|
||||||
)
|
)
|
||||||
if mm_encoder_only:
|
if mm_encoder_only:
|
||||||
model_config.multimodal_config.mm_encoder_only = True
|
model_config.multimodal_config.mm_encoder_only = True
|
||||||
|
|||||||
@@ -147,9 +147,15 @@ class Scheduler(SchedulerInterface):
|
|||||||
self.kv_events_config is not None
|
self.kv_events_config is not None
|
||||||
and self.kv_events_config.enable_kv_cache_events
|
and self.kv_events_config.enable_kv_cache_events
|
||||||
)
|
)
|
||||||
# Diffusion models may not sample any tokens for a denoising step.
|
# Pooling and diffusion models do not append a sampled token after
|
||||||
|
# prefill, so they should not reserve one context slot while scheduling.
|
||||||
self.num_sampled_tokens_per_step = (
|
self.num_sampled_tokens_per_step = (
|
||||||
1 if not vllm_config.model_config.is_diffusion else 0
|
0
|
||||||
|
if (
|
||||||
|
vllm_config.model_config.runner_type == "pooling"
|
||||||
|
or vllm_config.model_config.is_diffusion
|
||||||
|
)
|
||||||
|
else 1
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create KVConnector for the Scheduler. Note that each Worker
|
# Create KVConnector for the Scheduler. Note that each Worker
|
||||||
|
|||||||
Reference in New Issue
Block a user