[Bugfix][EPD] Skip sampling for encoder-only async steps (#58490)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-09-27 20:35:09 -07:00
committed by GitHub
parent 027b6f3a22
commit 53d2e16a97
3 changed files with 36 additions and 5 deletions
+33
View File
@@ -4,6 +4,7 @@
import copy import copy
import time import time
import uuid import uuid
from collections import deque
from concurrent.futures import Future, ThreadPoolExecutor from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import MagicMock, PropertyMock, patch from unittest.mock import MagicMock, PropertyMock, patch
@@ -395,6 +396,38 @@ def test_engine_core_concurrent_batches():
req_id = (req_id + 1) % 2 req_id = (req_id + 1) % 2
@pytest.mark.parametrize("encoder_only", [True, False])
def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only):
"""Encoder outputs are complete; only language-model steps need sampling."""
core = MagicMock()
core.batch_queue = deque()
core.batch_queue_size = 2
core.is_mm_encoder_only = encoder_only
core.is_pooling_model = False
core.scheduler.has_requests.side_effect = [True, False, False]
scheduled = core.scheduler.schedule.return_value
scheduled.total_num_scheduled_tokens = 16
scheduled.pending_structured_output_tokens = False
output = MagicMock()
executed: Future[ModelRunnerOutput | None] = Future()
executed.set_result(output if encoder_only else None)
sampled: Future[ModelRunnerOutput | None] = Future()
sampled.set_result(None if encoder_only else output)
core.model_executor.execute_model.return_value = executed
core.model_executor.sample_tokens.return_value = sampled
EngineCore.step_with_batch_queue(core)
if core.batch_queue:
EngineCore.step_with_batch_queue(core)
core.scheduler.update_from_output.assert_called_once_with(scheduled, output)
if encoder_only:
core.model_executor.sample_tokens.assert_not_called()
else:
core.model_executor.sample_tokens.assert_called_once()
@multi_gpu_test(num_gpus=2) @multi_gpu_test(num_gpus=2)
def test_engine_core_tp(): def test_engine_core_tp():
"""Test engine can initialize worker in tp properly.""" """Test engine can initialize worker in tp properly."""
@@ -99,6 +99,7 @@ def _run_engine_core_handshake(
), ),
speculative_config=None, speculative_config=None,
ec_transfer_config=None, ec_transfer_config=None,
is_mm_encoder_only=False,
max_concurrent_batches=1, max_concurrent_batches=1,
model_config=SimpleNamespace( model_config=SimpleNamespace(
runner_type="generate", runner_type="generate",
+2 -5
View File
@@ -219,10 +219,7 @@ class EngineCore:
logger.debug("Batch queue is enabled with size %d", self.batch_queue_size) logger.debug("Batch queue is enabled with size %d", self.batch_queue_size)
self.batch_queue = deque(maxlen=self.batch_queue_size) self.batch_queue = deque(maxlen=self.batch_queue_size)
self.is_ec_consumer = ( self.is_mm_encoder_only = vllm_config.is_mm_encoder_only
vllm_config.ec_transfer_config is None
or vllm_config.ec_transfer_config.is_ec_consumer
)
self.is_pooling_model = vllm_config.model_config.runner_type == "pooling" self.is_pooling_model = vllm_config.model_config.runner_type == "pooling"
self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None
@@ -702,7 +699,7 @@ class EngineCore:
exec_future = self.model_executor.execute_model( exec_future = self.model_executor.execute_model(
scheduler_output, non_block=True scheduler_output, non_block=True
) )
if self.is_ec_consumer: if not self.is_mm_encoder_only:
model_executed = scheduler_output.total_num_scheduled_tokens > 0 model_executed = scheduler_output.total_num_scheduled_tokens > 0
if self.is_pooling_model or not model_executed: if self.is_pooling_model or not model_executed: