[Refactor] Move iteration logging to the frontend (#46647)

Signed-off-by: maxyanghu <hyoung2991@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Shang Wang <shangw@nvidia.com>
This commit is contained in:
Max Hu
2026-07-15 17:59:05 -07:00
committed by GitHub
co-authored by Roger Wang Shang Wang
parent 9d1c695be5
commit 5a65ba5f17
9 changed files with 399 additions and 41 deletions
+82
View File
@@ -43,6 +43,65 @@ from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
pytestmark = pytest.mark.cpu_test
def test_make_scheduled_encoder_input_stats_output_embeddings():
scheduler = create_scheduler()
mm_features = [
MultiModalFeatureSpec(
data=MultiModalKwargsItem.dummy(),
modality="image",
identifier="image-0",
mm_position=PlaceholderRange(offset=0, length=196),
),
MultiModalFeatureSpec(
data=MultiModalKwargsItem.dummy(),
modality="video",
identifier="video-0",
mm_position=PlaceholderRange(offset=200, length=196),
),
MultiModalFeatureSpec(
data=MultiModalKwargsItem.dummy(),
modality="audio",
identifier="audio-0",
mm_position=PlaceholderRange(offset=400, length=49),
),
]
scheduler.requests["req"] = Mock(mm_features=mm_features)
stats = scheduler._make_scheduled_encoder_input_stats({"req": [0, 1, 2]})
assert stats is not None
assert stats.num_inputs == 3
assert stats.output_tokens == 441
def test_scheduled_encoder_input_stats_disabled_without_iteration_logging(
monkeypatch: pytest.MonkeyPatch,
):
scheduler = create_scheduler()
make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
scheduler_output = scheduler.schedule()
make_stats.assert_not_called()
assert scheduler_output.scheduled_encoder_input_stats is None
def test_scheduled_encoder_input_stats_disabled_without_log_stats(
monkeypatch: pytest.MonkeyPatch,
):
scheduler = create_scheduler()
scheduler.log_stats = False
scheduler.observability_config.enable_logging_iteration_details = True
make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
scheduler_output = scheduler.schedule()
make_stats.assert_not_called()
assert scheduler_output.scheduled_encoder_input_stats is None
def test_add_requests():
scheduler = create_scheduler()
requests = create_requests(num_requests=10)
@@ -108,6 +167,29 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
assert scheduler.running[i] == request
def test_scheduler_stats_route_to_existing_output_client():
scheduler = create_scheduler()
request = create_requests(num_requests=1)[0]
request.client_index = 1
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
model_output = ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[1000]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
)
engine_core_outputs = scheduler.update_from_output(scheduler_output, model_output)
assert 0 not in engine_core_outputs
assert engine_core_outputs[1].scheduler_stats is not None
assert len(engine_core_outputs[1].outputs) == 1
def test_schedule_multimodal_requests():
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_positions = [[PlaceholderRange(offset=i, length=100)] for i in range(10)]