Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -30,7 +30,7 @@ remove_docker_container
|
||||
docker pull "$IMAGE"
|
||||
|
||||
# Run the image
|
||||
docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE"
|
||||
docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE"
|
||||
|
||||
print_packages() {
|
||||
docker exec "$CONTAINER_NAME" bash -c "
|
||||
|
||||
@@ -99,7 +99,7 @@ rm -f "$build_log"
|
||||
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with'
|
||||
run_test() {
|
||||
local hf_offline=$1
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g "$IMAGE_NAME" \
|
||||
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import enum
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -19,6 +20,7 @@ from vllm.logger import (
|
||||
_DATE_FORMAT,
|
||||
_FORMAT,
|
||||
_configure_vllm_root_logger,
|
||||
_use_color,
|
||||
enable_trace_function_call,
|
||||
init_logger,
|
||||
)
|
||||
@@ -72,6 +74,27 @@ def test_default_vllm_root_logger_configuration(monkeypatch):
|
||||
assert formatter.datefmt == _DATE_FORMAT
|
||||
|
||||
|
||||
def test_use_color_force_color(monkeypatch):
|
||||
"""FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an
|
||||
explicit VLLM_LOGGING_COLOR=0 take precedence over it."""
|
||||
monkeypatch.setattr(sys, "stdout", io.StringIO())
|
||||
monkeypatch.setattr(sys, "stderr", io.StringIO())
|
||||
for var in ("NO_COLOR", "FORCE_COLOR", "VLLM_LOGGING_COLOR"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
assert not _use_color()
|
||||
|
||||
monkeypatch.setenv("FORCE_COLOR", "1")
|
||||
assert _use_color()
|
||||
|
||||
monkeypatch.setenv("VLLM_LOGGING_COLOR", "0")
|
||||
assert not _use_color()
|
||||
monkeypatch.delenv("VLLM_LOGGING_COLOR")
|
||||
|
||||
monkeypatch.setenv("NO_COLOR", "1")
|
||||
assert not _use_color()
|
||||
|
||||
|
||||
def test_descendent_loggers_depend_on_and_propagate_logs_to_root_logger(monkeypatch):
|
||||
"""This test presumes that VLLM_CONFIGURE_LOGGING (default: True) and
|
||||
VLLM_LOGGING_CONFIG_PATH (default: None) are not configured and default
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
@@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats():
|
||||
assert final_stats == expected_data
|
||||
|
||||
|
||||
def test_scheduler_ec_connector_stats():
|
||||
"""Test worker-side, scheduler-side, and combined EC connector stats."""
|
||||
|
||||
class GenericECConnectorStats(ECConnectorStats):
|
||||
def reset(self):
|
||||
self.data = {}
|
||||
|
||||
def aggregate(self, other: ECConnectorStats) -> ECConnectorStats:
|
||||
self.data.update(other.data)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
return {}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
|
||||
test_cases = (
|
||||
({"worker": 1}, None, {"worker": 1}),
|
||||
(None, {"scheduler": 2}, {"scheduler": 2}),
|
||||
({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}),
|
||||
)
|
||||
|
||||
for worker_data, scheduler_data, expected_data in test_cases:
|
||||
scheduler = create_scheduler()
|
||||
worker_stats = (
|
||||
GenericECConnectorStats(data=worker_data) if worker_data else None
|
||||
)
|
||||
scheduler_stats = (
|
||||
GenericECConnectorStats(data=scheduler_data) if scheduler_data else None
|
||||
)
|
||||
scheduler.ec_connector = Mock()
|
||||
scheduler.ec_connector.take_unavailable_requests.return_value = set()
|
||||
scheduler.ec_connector.get_ec_connector_stats.return_value = (
|
||||
scheduler_stats if worker_stats is None else None
|
||||
)
|
||||
|
||||
def update_connector_output(
|
||||
ec_connector_output: ECConnectorOutput,
|
||||
scheduler=scheduler,
|
||||
scheduler_stats=scheduler_stats,
|
||||
):
|
||||
scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats
|
||||
|
||||
scheduler.ec_connector.update_connector_output.side_effect = (
|
||||
update_connector_output
|
||||
)
|
||||
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=["req_0"],
|
||||
req_id_to_index={"req_0": 0},
|
||||
sampled_token_ids=[[123]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[None],
|
||||
ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats)
|
||||
if worker_stats
|
||||
else None,
|
||||
)
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=None,
|
||||
num_scheduled_tokens={"req_0": 1},
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_spec_decode_tokens={},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[0],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_output
|
||||
)
|
||||
|
||||
final_stats = next(
|
||||
iter(engine_core_outputs.values())
|
||||
).scheduler_stats.ec_connector_stats
|
||||
assert final_stats == expected_data
|
||||
|
||||
|
||||
def test_ec_connector_update_connector_output_called():
|
||||
"""Test that worker-side EC connector output is forwarded to the
|
||||
EC connector's update_connector_output hook."""
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
|
||||
|
||||
Covers only the base infra: no concrete connector type implements these
|
||||
hooks yet, so every assertion here is about the documented no-op defaults
|
||||
and the container contracts, mirroring the equivalent KV connector tests.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
|
||||
ECExampleConnector,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import (
|
||||
ECConnectorLogging,
|
||||
ECConnectorProm,
|
||||
ECConnectorStats,
|
||||
)
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_connector(tmp_path):
|
||||
config = Mock(spec=VllmConfig)
|
||||
config.ec_transfer_config = Mock()
|
||||
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
|
||||
config.ec_transfer_config.is_ec_producer = True
|
||||
config.ec_transfer_config.is_ec_consumer = False
|
||||
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
|
||||
|
||||
|
||||
def test_base_stats_methods_raise_not_implemented():
|
||||
"""The base container has no aggregation logic of its own."""
|
||||
stats = ECConnectorStats()
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reset()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.aggregate(stats)
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reduce()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.is_empty()
|
||||
|
||||
|
||||
def test_connector_hooks_default_to_none(example_connector):
|
||||
"""No connector type overrides these yet, so every hook is a no-op."""
|
||||
assert example_connector.get_ec_connector_stats() is None
|
||||
assert ECExampleConnector.build_ec_connector_stats() is None
|
||||
assert (
|
||||
ECExampleConnector.build_prom_metrics(
|
||||
vllm_config=Mock(spec=VllmConfig),
|
||||
metric_types={},
|
||||
labelnames=[],
|
||||
per_engine_labelvalues={},
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_ec_connector_output_is_empty_reflects_stats_field():
|
||||
output = ECConnectorOutput()
|
||||
assert output.is_empty()
|
||||
|
||||
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
|
||||
assert not output.is_empty()
|
||||
|
||||
|
||||
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
|
||||
"""A normal run with no EC connector configured must stay silent."""
|
||||
logging = ECConnectorLogging(ec_transfer_config=None)
|
||||
logging.log() # Should not raise, nothing to log.
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = None
|
||||
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
|
||||
assert prom.prom_metrics is None
|
||||
prom.observe({}) # Should not raise.
|
||||
@@ -5,6 +5,7 @@
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.v1.outputs import (
|
||||
@@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata):
|
||||
return FakeWorkerMeta(self.saves + other.saves)
|
||||
|
||||
|
||||
class FakeConnectorStats(ECConnectorStats):
|
||||
"""Records merge order, mirroring FakeWorkerMeta above."""
|
||||
|
||||
def __init__(self, saves: list[str]):
|
||||
self.saves = saves
|
||||
|
||||
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
|
||||
return FakeConnectorStats(self.saves + other.saves)
|
||||
|
||||
|
||||
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
|
||||
@@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
"""EC work done on any rank reaches the scheduler via output_rank's output.
|
||||
|
||||
The middle rank reports no worker metadata: it must neither seed nor clobber
|
||||
the accumulator.
|
||||
The middle rank reports no worker metadata or stats: it must neither seed
|
||||
nor clobber either accumulator.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(
|
||||
finished_sending={"mm0"},
|
||||
ec_connector_stats=FakeConnectorStats(["mm0"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
|
||||
)
|
||||
),
|
||||
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
|
||||
_worker_output(
|
||||
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
|
||||
ECConnectorOutput(
|
||||
ec_connector_stats=FakeConnectorStats(["mm2"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
|
||||
)
|
||||
),
|
||||
]
|
||||
|
||||
@@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
assert result is outputs[2]
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
assert result.ec_connector_output.finished_recving == {"mm1"}
|
||||
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
WORKER_META = object()
|
||||
CONNECTOR_STATS = object()
|
||||
|
||||
|
||||
def _scheduler_output() -> SimpleNamespace:
|
||||
@@ -38,6 +39,7 @@ def _connector(
|
||||
fake.is_consumer = is_consumer
|
||||
fake.get_finished.return_value = (None, None)
|
||||
fake.build_connector_worker_meta.return_value = WORKER_META
|
||||
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
|
||||
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
|
||||
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
|
||||
|
||||
@@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit():
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_connector_stats_are_reported_on_context_exit():
|
||||
"""Reported in the finally block, alongside the worker meta."""
|
||||
connector, fake = _connector()
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||
assert output.ec_connector_stats is None
|
||||
|
||||
assert output.ec_connector_stats is CONNECTOR_STATS
|
||||
|
||||
|
||||
def test_v1_producer_receives_existing_encoder_cache():
|
||||
encoder_cache = {"mm_cached": None}
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
|
||||
@@ -1452,3 +1452,41 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
|
||||
)
|
||||
|
||||
engine_core_proc.shutdown()
|
||||
|
||||
|
||||
def _ready_response_with_weight_transfer(weight_transfer_config):
|
||||
"""Build a ready response against a mock executor."""
|
||||
from vllm.v1.engine.core import EngineCoreProc
|
||||
|
||||
executor = MagicMock()
|
||||
executor.supports_draft_weight_updates.return_value = True
|
||||
|
||||
proc = MagicMock()
|
||||
proc.model_executor = executor
|
||||
proc.vllm_config = MagicMock()
|
||||
proc.vllm_config.weight_transfer_config = weight_transfer_config
|
||||
proc.vllm_config.lora_config = None
|
||||
proc.vllm_config.model_config.enable_sleep_mode = False
|
||||
proc.scheduler = MagicMock()
|
||||
|
||||
response = EngineCoreProc._make_ready_response(proc)
|
||||
return response, executor
|
||||
|
||||
|
||||
def test_ready_response_skips_draft_rpc_without_weight_transfer():
|
||||
"""No weight-transfer config: the executor must not be consulted."""
|
||||
response, executor = _ready_response_with_weight_transfer(None)
|
||||
|
||||
executor.supports_draft_weight_updates.assert_not_called()
|
||||
assert response.supports_draft_weight_updates is False
|
||||
|
||||
|
||||
def test_ready_response_queries_executor_with_weight_transfer():
|
||||
"""Weight transfer configured: the executor is still consulted."""
|
||||
weight_transfer_config = MagicMock()
|
||||
weight_transfer_config.backend = "nixl"
|
||||
|
||||
response, executor = _ready_response_with_weight_transfer(weight_transfer_config)
|
||||
|
||||
executor.supports_draft_weight_updates.assert_called_once()
|
||||
assert response.supports_draft_weight_updates is True
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
SchedulerOffloadConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
@@ -32,6 +33,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
@@ -876,6 +879,43 @@ def test_canonical_layout_certifies_v2_model_runner():
|
||||
assert _parallelism_agnostic(groups, canonical=True, v2=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("scheduler", [False, True])
|
||||
def test_canonical_mla_dsa_rows_are_tp_independent(tp_size, scheduler):
|
||||
specs = {
|
||||
f"l{i}": _mla_spec(block_size=64, head_size=size, dtype=torch.uint8)
|
||||
for i, size in enumerate([656] * 4 + [132] * 3)
|
||||
}
|
||||
uniform = UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * uniform.page_size_bytes,
|
||||
layers=list(specs),
|
||||
layer_stride=0,
|
||||
block_stride=uniform.page_size_bytes,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(list(specs), uniform)],
|
||||
)
|
||||
if scheduler:
|
||||
kv_cache_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
config = _make_vllm_config(
|
||||
tensor_parallel_size=tp_size,
|
||||
extra_config={"canonical_layout": True, "cpu_bytes_to_use": 196608 * 8},
|
||||
)
|
||||
config.cache_config.kv_cache_layout = "LBHNC"
|
||||
config.parallel_config.distributed_executor_backend = "mp"
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
spec = TieringOffloadingSpec(offloading_config)
|
||||
assert spec.kv_bytes_per_chunk == 196608
|
||||
assert spec.num_chunks == 8
|
||||
mapper = FileMapper.from_offloading_spec("/cache", spec, parallel_agnostic=True)
|
||||
assert mapper.get_run_config()["tp_size"] == 1
|
||||
assert mapper.get_run_config()["replicated_layout"] is True
|
||||
|
||||
|
||||
def test_parallelism_agnostic_disabled_on_v2_model_runner():
|
||||
config = _make_vllm_config()
|
||||
config.use_v2_model_runner = True
|
||||
|
||||
@@ -97,6 +97,7 @@ def _make_worker(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
replicated_layout: bool = False,
|
||||
rank: int = 0,
|
||||
canonical_layout: bool = False,
|
||||
):
|
||||
"""Create an OffloadingConnectorWorker with mocked dependencies."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
|
||||
@@ -106,6 +107,7 @@ def _make_worker(
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.replicated_layout = replicated_layout
|
||||
spec.config = MagicMock()
|
||||
spec.config.canonical_layout = canonical_layout
|
||||
spec.config.parallel.rank = rank
|
||||
spec.get_worker.return_value = MagicMock()
|
||||
|
||||
@@ -204,11 +206,13 @@ def test_prepare_store_kv_non_writer_marks_completed_without_submit():
|
||||
assert meta.completed_jobs == {7: 1}
|
||||
|
||||
|
||||
def test_prepare_store_kv_writer_submits_store():
|
||||
@pytest.mark.parametrize("rank,canonical_layout", [(0, False), (0, True), (1, True)])
|
||||
def test_prepare_store_kv_writer_submits_store(rank, canonical_layout):
|
||||
worker, _ = _make_worker(
|
||||
KVCacheConfig(num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]),
|
||||
replicated_layout=True,
|
||||
rank=0,
|
||||
rank=rank,
|
||||
canonical_layout=canonical_layout,
|
||||
)
|
||||
|
||||
worker.prepare_store_kv(_store_metadata(8))
|
||||
|
||||
@@ -35,6 +35,12 @@ from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import (
|
||||
ECConnectorPromMetrics,
|
||||
ECConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -219,6 +225,17 @@ class ECConnectorBase(ABC):
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_ec_connector_stats(self) -> "ECConnectorStats | None":
|
||||
"""
|
||||
Get the EC connector stats collected during the last interval.
|
||||
|
||||
Callable on either a worker-side or scheduler-side connector
|
||||
instance: the worker-side instance reports stats gathered during
|
||||
cache save/load, while the scheduler-side instance reports stats
|
||||
gathered during scheduling decisions.
|
||||
"""
|
||||
return None
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
@@ -336,3 +353,29 @@ class ECConnectorBase(ABC):
|
||||
leave this as False.
|
||||
"""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def build_ec_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> "ECConnectorStats | None":
|
||||
"""
|
||||
ECConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own ECConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> "ECConnectorPromMetrics | None":
|
||||
"""
|
||||
Create an ECConnectorPromMetrics subclass which should register
|
||||
per-connector Prometheus metrics and implement observe() to
|
||||
expose connector transfer stats via Prometheus.
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, TypeAlias, TypeVar
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from vllm.config import ECTransferConfig, VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
from vllm.logger import init_logger
|
||||
|
||||
PromMetric: TypeAlias = Gauge | Counter | Histogram
|
||||
PromMetricT = TypeVar("PromMetricT", bound=PromMetric)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECConnectorStats:
|
||||
"""
|
||||
Base class for EC Connector Stats, a container for transfer performance
|
||||
metrics or otherwise important telemetry from the connector.
|
||||
All sub-classes need to be serializable as stats are sent from worker to
|
||||
logger process.
|
||||
"""
|
||||
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def reset(self):
|
||||
"""Reset the stats, clear the state."""
|
||||
raise NotImplementedError
|
||||
|
||||
def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats":
|
||||
"""
|
||||
Aggregate stats with another `ECConnectorStats` object.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""
|
||||
Reduce the observations collected during a time interval to one or
|
||||
more representative values (eg avg/median/sum of the series).
|
||||
This is meant to be called by the logger to produce a summary of the
|
||||
stats for the last time interval.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
"""Return True if the stats are empty."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class ECConnectorLogging:
|
||||
def __init__(self, ec_transfer_config: ECTransferConfig | None):
|
||||
# Instantiate the connector's stats class.
|
||||
if ec_transfer_config and ec_transfer_config.ec_connector:
|
||||
self.connector_cls = ECConnectorFactory.get_connector_class(
|
||||
ec_transfer_config
|
||||
)
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.transfer_stats_accumulator: ECConnectorStats | None = None
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any]):
|
||||
# Should not be called when an ECConnector is not configured.
|
||||
assert self.connector_cls is not None
|
||||
# Called periodically when connector syncs with the scheduler.
|
||||
# Note that this is not the same as the logging interval.
|
||||
# We expect transfer_stats_data to be aggregated across all workers and
|
||||
# consist of observations from a single connector.
|
||||
transfer_stats = self.connector_cls.build_ec_connector_stats(
|
||||
transfer_stats_data
|
||||
)
|
||||
if transfer_stats is None:
|
||||
logger.warning_once(
|
||||
"The connector %s is collecting stats but "
|
||||
"does not implement the "
|
||||
"`build_ec_connector_stats` method. "
|
||||
"Stats will not be logged.",
|
||||
self.connector_cls,
|
||||
)
|
||||
return
|
||||
|
||||
if self.transfer_stats_accumulator is None:
|
||||
self.transfer_stats_accumulator = transfer_stats
|
||||
else:
|
||||
# Accumulate last interval stats.
|
||||
self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate(
|
||||
transfer_stats
|
||||
)
|
||||
|
||||
def log(self, log_fn=logger.info):
|
||||
"""Log transfer metrics periodically, similar to throughput logging"""
|
||||
if (
|
||||
self.transfer_stats_accumulator
|
||||
and not self.transfer_stats_accumulator.is_empty()
|
||||
):
|
||||
# Produce a single cumulative stats object for the last time
|
||||
# interval from the recorded observations.
|
||||
xfer_metrics = self.transfer_stats_accumulator.reduce()
|
||||
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
|
||||
log_fn("EC Transfer metrics: %s", xfer_metrics_str)
|
||||
|
||||
# Reset metrics for next interval
|
||||
self.reset()
|
||||
|
||||
|
||||
class ECConnectorPromMetrics:
|
||||
"""
|
||||
A base class for per-connector Prometheus metric registration
|
||||
and recording.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
self._ec_transfer_config = vllm_config.ec_transfer_config
|
||||
self._gauge_cls = metric_types[Gauge]
|
||||
self._counter_cls = metric_types[Counter]
|
||||
self._histogram_cls = metric_types[Histogram]
|
||||
self._labelnames = labelnames
|
||||
self.per_engine_labelvalues = per_engine_labelvalues
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""
|
||||
Record the supplied transfer statistics to Prometheus metrics. These
|
||||
statistics are engine-specific, and should be recorded to a metric
|
||||
with the appropriate 'engine' label. These metric instances can be
|
||||
created using the create_metric_per_engine() helper method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class ECConnectorProm:
|
||||
"""
|
||||
Support for registering per-connector Prometheus metrics, and
|
||||
recording transfer statistics to those metrics. Uses
|
||||
ECConnectorBase.build_prom_metrics().
|
||||
"""
|
||||
|
||||
_gauge_cls = Gauge
|
||||
_counter_cls = Counter
|
||||
_histogram_cls = Histogram
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
self.prom_metrics: ECConnectorPromMetrics | None = None
|
||||
ec_transfer_config = vllm_config.ec_transfer_config
|
||||
if ec_transfer_config and ec_transfer_config.ec_connector:
|
||||
connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config)
|
||||
metric_types = {
|
||||
Gauge: self._gauge_cls,
|
||||
Counter: self._counter_cls,
|
||||
Histogram: self._histogram_cls,
|
||||
}
|
||||
self.prom_metrics = connector_cls.build_prom_metrics(
|
||||
vllm_config,
|
||||
metric_types,
|
||||
labelnames,
|
||||
per_engine_labelvalues,
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
if self.prom_metrics is None:
|
||||
return
|
||||
self.prom_metrics.observe(transfer_stats_data, engine_idx)
|
||||
@@ -103,6 +103,7 @@ class ECOutputAggregator:
|
||||
finished_sending = set[str]()
|
||||
finished_recving = set[str]()
|
||||
worker_meta = None
|
||||
connector_stats = None
|
||||
for model_runner_output in outputs:
|
||||
assert model_runner_output is not None
|
||||
ec_output = model_runner_output.ec_connector_output
|
||||
@@ -117,9 +118,17 @@ class ECOutputAggregator:
|
||||
meta if worker_meta is None else worker_meta.aggregate(meta)
|
||||
)
|
||||
|
||||
if stats := ec_output.ec_connector_stats:
|
||||
connector_stats = (
|
||||
stats
|
||||
if connector_stats is None
|
||||
else connector_stats.aggregate(stats)
|
||||
)
|
||||
|
||||
aggregated = ECConnectorOutput(
|
||||
finished_sending=finished_sending or None,
|
||||
finished_recving=finished_recving or None,
|
||||
ec_connector_stats=connector_stats,
|
||||
ec_connector_worker_meta=worker_meta,
|
||||
)
|
||||
if aggregated.is_empty():
|
||||
|
||||
@@ -233,6 +233,20 @@ def build_offloading_config(
|
||||
and parallel_config.world_size == tp_size
|
||||
)
|
||||
|
||||
if canonical_layout and is_parallelism_agnostic:
|
||||
replicated_layout = (
|
||||
all(
|
||||
type(spec) is MLAAttentionSpec
|
||||
for _, group in selected_groups
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
and (
|
||||
parallel_config.world_size == 1
|
||||
or parallel_config.distributed_executor_backend == "mp"
|
||||
)
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
cache_dtype = (
|
||||
vllm_config.model_config.dtype
|
||||
|
||||
@@ -53,7 +53,9 @@ class OffloadingConnectorWorker:
|
||||
self.worker: OffloadingWorker | None = None
|
||||
# Non-writers still ack: pending_count waits for world_size per job.
|
||||
self._is_store_writer = (
|
||||
not self.spec.replicated_layout or self.spec.config.parallel.rank == 0
|
||||
not self.spec.replicated_layout
|
||||
or self.spec.config.canonical_layout
|
||||
or self.spec.config.parallel.rank == 0
|
||||
)
|
||||
|
||||
# job_id -> req_id for in-flight loads.
|
||||
|
||||
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
|
||||
VLLM_LOGGING_CONFIG_PATH: str | None = None
|
||||
VLLM_LOGGING_COLOR: str = "auto"
|
||||
NO_COLOR: bool = False
|
||||
FORCE_COLOR: bool = False
|
||||
VLLM_LOG_STATS_INTERVAL: float = 10.0
|
||||
VLLM_TRACE_FUNCTION: int = 0
|
||||
VLLM_USE_FLASHINFER_SAMPLER: bool = True
|
||||
@@ -840,6 +841,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_LOGGING_COLOR": lambda: os.getenv("VLLM_LOGGING_COLOR", "auto"),
|
||||
# Standard unix flag for disabling ANSI color codes
|
||||
"NO_COLOR": lambda: os.getenv("NO_COLOR", "0") != "0",
|
||||
# De-facto standard flag for forcing ANSI color codes (e.g. non-tty case)
|
||||
"FORCE_COLOR": lambda: os.getenv("FORCE_COLOR", "0") != "0",
|
||||
# If set, vllm will log stats at this interval in seconds
|
||||
# If not set, vllm will log stats every 10 seconds.
|
||||
"VLLM_LOG_STATS_INTERVAL": lambda: (
|
||||
@@ -2335,6 +2338,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"NO_COLOR",
|
||||
"VLLM_SKIP_VERSION_SUFFIX",
|
||||
"FORCE_COLOR",
|
||||
}
|
||||
|
||||
from vllm.config.utils import normalize_value
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ _DATE_FORMAT = "%m-%d %H:%M:%S"
|
||||
def _use_color() -> bool:
|
||||
if envs.NO_COLOR or envs.VLLM_LOGGING_COLOR == "0":
|
||||
return False
|
||||
if envs.VLLM_LOGGING_COLOR == "1":
|
||||
if envs.VLLM_LOGGING_COLOR == "1" or envs.FORCE_COLOR:
|
||||
return True
|
||||
if envs.VLLM_LOGGING_STREAM == "ext://sys.stdout": # stdout
|
||||
return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
@@ -2243,6 +2244,23 @@ class Scheduler(SchedulerInterface):
|
||||
else scheduler_kv_connector_stats
|
||||
)
|
||||
|
||||
# Worker-side EC connector stats from the model runner output.
|
||||
ec_connector_stats: ECConnectorStats | None = (
|
||||
ec_connector_output.ec_connector_stats if ec_connector_output else None
|
||||
)
|
||||
if self.ec_connector:
|
||||
# Scheduler-side EC connector stats collected after connector update.
|
||||
scheduler_ec_connector_stats = self.ec_connector.get_ec_connector_stats()
|
||||
if (
|
||||
scheduler_ec_connector_stats is not None
|
||||
and not scheduler_ec_connector_stats.is_empty()
|
||||
):
|
||||
ec_connector_stats = (
|
||||
ec_connector_stats.aggregate(scheduler_ec_connector_stats)
|
||||
if ec_connector_stats is not None
|
||||
else scheduler_ec_connector_stats
|
||||
)
|
||||
|
||||
# collect KV cache events from KV cache manager
|
||||
events = self.kv_cache_manager.take_events()
|
||||
|
||||
@@ -2287,6 +2305,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_connector_stats,
|
||||
cudagraph_stats,
|
||||
perf_stats,
|
||||
ec_connector_stats=ec_connector_stats,
|
||||
)
|
||||
) is not None:
|
||||
# Return stats to only one of the front-ends.
|
||||
@@ -2772,6 +2791,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_connector_stats: KVConnectorStats | None = None,
|
||||
cudagraph_stats: CUDAGraphStat | None = None,
|
||||
perf_stats: PerfStats | None = None,
|
||||
ec_connector_stats: ECConnectorStats | None = None,
|
||||
) -> SchedulerStats | None:
|
||||
if not self.log_stats:
|
||||
return None
|
||||
@@ -2790,6 +2810,9 @@ class Scheduler(SchedulerInterface):
|
||||
connector_stats_payload = (
|
||||
kv_connector_stats.to_dict() if kv_connector_stats else None
|
||||
)
|
||||
ec_connector_stats_payload = (
|
||||
ec_connector_stats.data if ec_connector_stats else None
|
||||
)
|
||||
return SchedulerStats(
|
||||
num_running_reqs=len(self.running),
|
||||
num_waiting_reqs=len(self.waiting),
|
||||
@@ -2802,6 +2825,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_connector_stats=connector_stats_payload,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
perf_stats=perf_stats,
|
||||
ec_connector_stats=ec_connector_stats_payload,
|
||||
)
|
||||
|
||||
def make_spec_decoding_stats(
|
||||
|
||||
@@ -1691,6 +1691,8 @@ class EngineCoreProc(EngineCore):
|
||||
enable_sleep_mode=self.vllm_config.model_config.enable_sleep_mode,
|
||||
supports_draft_weight_updates=(
|
||||
self.model_executor.supports_draft_weight_updates()
|
||||
if self.vllm_config.weight_transfer_config is not None
|
||||
else False
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@ from prometheus_client import Counter, Gauge, Histogram
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.cuda_graph import CUDAGraphLogging
|
||||
from vllm.config import SupportsMetricsInfo, VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import (
|
||||
ECConnectorLogging,
|
||||
ECConnectorProm,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorLogging,
|
||||
KVConnectorProm,
|
||||
@@ -118,6 +122,8 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
self.spec_decoding_logging = SpecDecodingLogging(is_diffusion=is_diffusion)
|
||||
kv_transfer_config = self.vllm_config.kv_transfer_config
|
||||
self.kv_connector_logging = KVConnectorLogging(kv_transfer_config)
|
||||
ec_transfer_config = self.vllm_config.ec_transfer_config
|
||||
self.ec_connector_logging = ECConnectorLogging(ec_transfer_config)
|
||||
self.cudagraph_logging = None
|
||||
if self.vllm_config.observability_config.cudagraph_metrics:
|
||||
self.cudagraph_logging = CUDAGraphLogging(
|
||||
@@ -224,6 +230,8 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats)
|
||||
if kv_connector_stats := scheduler_stats.kv_connector_stats:
|
||||
self.kv_connector_logging.observe(kv_connector_stats)
|
||||
if ec_connector_stats := scheduler_stats.ec_connector_stats:
|
||||
self.ec_connector_logging.observe(ec_connector_stats)
|
||||
if (
|
||||
self.cudagraph_logging is not None
|
||||
and scheduler_stats.cudagraph_stats is not None
|
||||
@@ -319,6 +327,7 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
|
||||
self.spec_decoding_logging.log(log_fn=log_fn)
|
||||
self.kv_connector_logging.log(log_fn=log_fn)
|
||||
self.ec_connector_logging.log(log_fn=log_fn)
|
||||
if self.cudagraph_logging is not None:
|
||||
self.cudagraph_logging.log(log_fn=log_fn)
|
||||
if self._enable_perf_stats():
|
||||
@@ -451,6 +460,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
_histogram_cls = Histogram
|
||||
_spec_decoding_cls = SpecDecodingProm
|
||||
_kv_connector_cls = KVConnectorProm
|
||||
_ec_connector_cls = ECConnectorProm
|
||||
_perf_metrics_cls = PerfMetricsProm
|
||||
|
||||
def __init__(
|
||||
@@ -488,6 +498,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
self.kv_connector_prom = self._kv_connector_cls(
|
||||
vllm_config, labelnames, per_engine_labelvalues
|
||||
)
|
||||
self.ec_connector_prom = self._ec_connector_cls(
|
||||
vllm_config, labelnames, per_engine_labelvalues
|
||||
)
|
||||
self.perf_metrics_prom = self._perf_metrics_cls(
|
||||
vllm_config, labelnames, per_engine_labelvalues
|
||||
)
|
||||
@@ -1064,6 +1077,11 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
scheduler_stats.kv_connector_stats, engine_idx
|
||||
)
|
||||
|
||||
if scheduler_stats.ec_connector_stats is not None:
|
||||
self.ec_connector_prom.observe(
|
||||
scheduler_stats.ec_connector_stats, engine_idx
|
||||
)
|
||||
|
||||
if scheduler_stats.perf_stats is not None:
|
||||
self.perf_metrics_prom.observe(scheduler_stats.perf_stats, engine_idx)
|
||||
|
||||
|
||||
@@ -205,6 +205,7 @@ class SchedulerStats:
|
||||
|
||||
spec_decoding_stats: SpecDecodingStats | None = None
|
||||
kv_connector_stats: dict[str, Any] | None = None
|
||||
ec_connector_stats: dict[str, Any] | None = None
|
||||
|
||||
waiting_lora_adapters: dict[str, int] = field(default_factory=dict)
|
||||
running_lora_adapters: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_events import KVConnectorKVEvents
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorWorkerMetadata,
|
||||
@@ -307,12 +308,14 @@ class ECConnectorOutput:
|
||||
# [mm_hash]
|
||||
finished_sending: set[str] | None = None
|
||||
finished_recving: set[str] | None = None
|
||||
ec_connector_stats: "ECConnectorStats | None" = None
|
||||
ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None
|
||||
|
||||
def is_empty(self):
|
||||
return (
|
||||
not self.finished_sending
|
||||
and not self.finished_recving
|
||||
and not self.ec_connector_stats
|
||||
and not self.ec_connector_worker_meta
|
||||
)
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ class ActiveECConnector(ECConnector):
|
||||
output.finished_sending, output.finished_recving = (
|
||||
ec_connector.get_finished(scheduler_output.finished_req_ids)
|
||||
)
|
||||
output.ec_connector_stats = ec_connector.get_ec_connector_stats()
|
||||
output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta()
|
||||
ec_connector.clear_connector_metadata()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user