[Refactor] Move iteration logging to the frontend (#46647)
Signed-off-by: maxyanghu <hyoung2991@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Shang Wang <shangw@nvidia.com>
This commit is contained in:
co-authored by
Roger Wang
Shang Wang
parent
9d1c695be5
commit
5a65ba5f17
@@ -43,6 +43,65 @@ from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_make_scheduled_encoder_input_stats_output_embeddings():
|
||||
scheduler = create_scheduler()
|
||||
mm_features = [
|
||||
MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="image",
|
||||
identifier="image-0",
|
||||
mm_position=PlaceholderRange(offset=0, length=196),
|
||||
),
|
||||
MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="video",
|
||||
identifier="video-0",
|
||||
mm_position=PlaceholderRange(offset=200, length=196),
|
||||
),
|
||||
MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="audio",
|
||||
identifier="audio-0",
|
||||
mm_position=PlaceholderRange(offset=400, length=49),
|
||||
),
|
||||
]
|
||||
scheduler.requests["req"] = Mock(mm_features=mm_features)
|
||||
|
||||
stats = scheduler._make_scheduled_encoder_input_stats({"req": [0, 1, 2]})
|
||||
|
||||
assert stats is not None
|
||||
assert stats.num_inputs == 3
|
||||
assert stats.output_tokens == 441
|
||||
|
||||
|
||||
def test_scheduled_encoder_input_stats_disabled_without_iteration_logging(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
scheduler = create_scheduler()
|
||||
make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
|
||||
monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
make_stats.assert_not_called()
|
||||
assert scheduler_output.scheduled_encoder_input_stats is None
|
||||
|
||||
|
||||
def test_scheduled_encoder_input_stats_disabled_without_log_stats(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
scheduler = create_scheduler()
|
||||
scheduler.log_stats = False
|
||||
scheduler.observability_config.enable_logging_iteration_details = True
|
||||
make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
|
||||
monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
make_stats.assert_not_called()
|
||||
assert scheduler_output.scheduled_encoder_input_stats is None
|
||||
|
||||
|
||||
def test_add_requests():
|
||||
scheduler = create_scheduler()
|
||||
requests = create_requests(num_requests=10)
|
||||
@@ -108,6 +167,29 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
|
||||
assert scheduler.running[i] == request
|
||||
|
||||
|
||||
def test_scheduler_stats_route_to_existing_output_client():
|
||||
scheduler = create_scheduler()
|
||||
request = create_requests(num_requests=1)[0]
|
||||
request.client_index = 1
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[1000]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
engine_core_outputs = scheduler.update_from_output(scheduler_output, model_output)
|
||||
|
||||
assert 0 not in engine_core_outputs
|
||||
assert engine_core_outputs[1].scheduler_stats is not None
|
||||
assert len(engine_core_outputs[1].outputs) == 1
|
||||
|
||||
|
||||
def test_schedule_multimodal_requests():
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_positions = [[PlaceholderRange(offset=i, length=100)] for i in range(10)]
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.engine import EngineCoreOutputs
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
|
||||
|
||||
|
||||
class FakeEngineCore:
|
||||
def _make_iteration_details_stats(
|
||||
self, iteration_details: SchedulerIterationDetails
|
||||
) -> SchedulerStats:
|
||||
return SchedulerStats(iteration_details=iteration_details)
|
||||
|
||||
|
||||
def make_iteration_details() -> SchedulerIterationDetails:
|
||||
return SchedulerIterationDetails(
|
||||
iteration_index=1,
|
||||
num_ctx_requests=2,
|
||||
num_ctx_tokens=3,
|
||||
num_generation_requests=4,
|
||||
num_generation_tokens=5,
|
||||
elapsed_ms=6.7,
|
||||
)
|
||||
|
||||
|
||||
def make_fake_engine(log_stats: bool = True) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
log_stats=log_stats,
|
||||
vllm_config=SimpleNamespace(
|
||||
observability_config=SimpleNamespace(
|
||||
enable_logging_iteration_details=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_capture_iteration_details_disabled_without_log_stats():
|
||||
engine = make_fake_engine(log_stats=False)
|
||||
|
||||
with EngineCore.capture_iteration_details(engine, None) as iteration_details:
|
||||
assert iteration_details is None
|
||||
|
||||
assert not hasattr(engine, "_iteration_index")
|
||||
|
||||
|
||||
def test_capture_iteration_details_fills_elapsed_time():
|
||||
engine = make_fake_engine()
|
||||
|
||||
with EngineCore.capture_iteration_details(engine, None) as iteration_details:
|
||||
assert iteration_details is not None
|
||||
assert iteration_details.elapsed_ms == 0.0
|
||||
assert iteration_details.is_dummy
|
||||
time.sleep(0.001)
|
||||
|
||||
assert iteration_details is not None
|
||||
assert iteration_details.elapsed_ms > 0.0
|
||||
assert engine._iteration_index == 1
|
||||
|
||||
|
||||
def test_attach_iteration_details_uses_existing_output():
|
||||
iteration_details = make_iteration_details()
|
||||
outputs = {
|
||||
2: EngineCoreOutputs(scheduler_stats=SchedulerStats()),
|
||||
1: EngineCoreOutputs(scheduler_stats=SchedulerStats()),
|
||||
}
|
||||
|
||||
EngineCore._attach_iteration_details(FakeEngineCore(), outputs, iteration_details)
|
||||
|
||||
assert 0 not in outputs
|
||||
assert outputs[2].scheduler_stats is not None
|
||||
assert outputs[2].scheduler_stats.iteration_details == iteration_details
|
||||
assert outputs[1].scheduler_stats is not None
|
||||
assert outputs[1].scheduler_stats.iteration_details is None
|
||||
|
||||
|
||||
def test_attach_iteration_details_falls_back_to_client_zero_without_outputs():
|
||||
iteration_details = make_iteration_details()
|
||||
outputs: dict[int, EngineCoreOutputs] = {}
|
||||
|
||||
EngineCore._attach_iteration_details(FakeEngineCore(), outputs, iteration_details)
|
||||
|
||||
assert set(outputs) == {0}
|
||||
assert outputs[0].scheduler_stats is not None
|
||||
assert outputs[0].scheduler_stats.iteration_details == iteration_details
|
||||
@@ -1,12 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.core.sched.output import ScheduledEncoderInputStats, SchedulerOutput
|
||||
from vllm.v1.engine import EngineCoreOutputs, FinishReason
|
||||
from vllm.v1.metrics.stats import (
|
||||
IterationStats,
|
||||
PrefillStats,
|
||||
PromptTokenStats,
|
||||
RequestStateStats,
|
||||
SchedulerIterationDetails,
|
||||
SchedulerStats,
|
||||
)
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
from vllm.v1.utils import compute_iteration_details
|
||||
|
||||
|
||||
def test_iteration_stats_repr():
|
||||
@@ -14,6 +19,45 @@ def test_iteration_stats_repr():
|
||||
assert repr(iteration_stats).startswith("IterationStats(")
|
||||
|
||||
|
||||
def test_scheduler_iteration_details_serialization():
|
||||
iteration_details = SchedulerIterationDetails(
|
||||
iteration_index=1,
|
||||
num_ctx_requests=2,
|
||||
num_ctx_tokens=3,
|
||||
num_generation_requests=4,
|
||||
num_generation_tokens=5,
|
||||
elapsed_ms=6.7,
|
||||
num_encoder_inputs=2,
|
||||
num_encoder_output_tokens=392,
|
||||
)
|
||||
outputs = EngineCoreOutputs(
|
||||
scheduler_stats=SchedulerStats(
|
||||
kv_cache_usage=0.5,
|
||||
iteration_details=iteration_details,
|
||||
)
|
||||
)
|
||||
|
||||
encoded = MsgpackEncoder().encode(outputs)
|
||||
decoded = MsgpackDecoder(EngineCoreOutputs).decode(encoded)
|
||||
|
||||
assert decoded.scheduler_stats is not None
|
||||
assert decoded.scheduler_stats.kv_cache_usage == 0.5
|
||||
assert decoded.scheduler_stats.iteration_details == iteration_details
|
||||
|
||||
|
||||
def test_compute_iteration_details_includes_encoder_stats():
|
||||
scheduler_output = SchedulerOutput.make_empty()
|
||||
scheduler_output.scheduled_encoder_input_stats = ScheduledEncoderInputStats(
|
||||
num_inputs=2,
|
||||
output_tokens=392,
|
||||
)
|
||||
|
||||
iteration_details = compute_iteration_details(scheduler_output)
|
||||
|
||||
assert iteration_details.num_encoder_inputs == 2
|
||||
assert iteration_details.num_encoder_output_tokens == 392
|
||||
|
||||
|
||||
def test_prefill_kv_computed_with_cache():
|
||||
"""Test that prefill KV compute correctly excludes cached tokens."""
|
||||
iteration_stats = IterationStats()
|
||||
|
||||
Reference in New Issue
Block a user