[Bugfix] Fix np.float64 leaking into the KV transfer metrics log (#57068)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -8,6 +8,7 @@
|
|||||||
import os
|
import os
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -218,6 +219,9 @@ class TestHF3FSKVConnectorStats:
|
|||||||
assert result["Num save task success"] == pytest.approx(2.0, rel=0.01)
|
assert result["Num save task success"] == pytest.approx(2.0, rel=0.01)
|
||||||
assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01)
|
assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01)
|
||||||
assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01)
|
assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01)
|
||||||
|
# Reduced values must be plain Python scalars so CLI logging renders
|
||||||
|
# them without numpy reprs (eg np.float64(...)).
|
||||||
|
assert all(not isinstance(v, np.generic) for v in result.values())
|
||||||
|
|
||||||
def test_clone_and_reset(self, hf3fs_stats):
|
def test_clone_and_reset(self, hf3fs_stats):
|
||||||
"""clone_and_reset() returns a copy with data and resets the original."""
|
"""clone_and_reset() returns a copy with data and resets the original."""
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
import threading
|
import threading
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||||
MooncakeConnector,
|
MooncakeConnector,
|
||||||
MooncakeConnectorWorker,
|
MooncakeConnectorWorker,
|
||||||
@@ -40,6 +42,9 @@ def test_record_transfer_and_reduce():
|
|||||||
assert reduced["Num failed transfers"] == 0
|
assert reduced["Num failed transfers"] == 0
|
||||||
assert reduced["Num failed recvs"] == 0
|
assert reduced["Num failed recvs"] == 0
|
||||||
assert reduced["Num KV expired reqs"] == 0
|
assert reduced["Num KV expired reqs"] == 0
|
||||||
|
# Reduced values must be plain Python scalars so CLI logging renders
|
||||||
|
# them without numpy reprs (eg np.float64(...)).
|
||||||
|
assert all(not isinstance(v, np.generic) for v in reduced.values())
|
||||||
|
|
||||||
|
|
||||||
def test_record_failures_keeps_stats_non_empty():
|
def test_record_failures_keeps_stats_non_empty():
|
||||||
|
|||||||
@@ -20,7 +20,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
|||||||
SupportsHMA,
|
SupportsHMA,
|
||||||
supports_hma,
|
supports_hma,
|
||||||
)
|
)
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||||
|
KVConnectorLogging,
|
||||||
|
KVConnectorStats,
|
||||||
|
)
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
||||||
MultiConnector,
|
MultiConnector,
|
||||||
MultiKVConnectorPromMetrics,
|
MultiKVConnectorPromMetrics,
|
||||||
@@ -817,6 +820,36 @@ class TestMultiConnectorStats:
|
|||||||
assert "Num successful transfers" in reduced["NixlConnector"]
|
assert "Num successful transfers" in reduced["NixlConnector"]
|
||||||
assert reduced["NixlConnector"]["Num successful transfers"] == 2
|
assert reduced["NixlConnector"]["Num successful transfers"] == 2
|
||||||
|
|
||||||
|
def test_log_renders_plain_scalars(self):
|
||||||
|
"""Reduced stats must render as plain numbers in the CLI log, not
|
||||||
|
numpy reprs (eg np.float64(2.338))."""
|
||||||
|
stats = MultiKVConnectorStats(
|
||||||
|
data={
|
||||||
|
"NixlConnector": NixlKVConnectorStats(
|
||||||
|
data={
|
||||||
|
"transfer_duration": [1.0, 2.0],
|
||||||
|
"post_duration": [0.1, 0.2],
|
||||||
|
"bytes_transferred": [1024, 2048],
|
||||||
|
"num_descriptors": [10, 20],
|
||||||
|
"num_failed_transfers": [],
|
||||||
|
"num_failed_notifications": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
kv_logging = KVConnectorLogging(kv_transfer_config=None)
|
||||||
|
kv_logging.transfer_stats_accumulator = stats
|
||||||
|
log_records: list[tuple] = []
|
||||||
|
kv_logging.log(log_fn=lambda *args: log_records.append(args))
|
||||||
|
|
||||||
|
assert len(log_records) == 1
|
||||||
|
msg = log_records[0][1]
|
||||||
|
assert "np.float" not in msg
|
||||||
|
assert "'Avg xfer time (ms)': 1500.0" in msg
|
||||||
|
# The accumulator is reset after logging.
|
||||||
|
assert kv_logging.transfer_stats_accumulator is None
|
||||||
|
|
||||||
def test_reset(self):
|
def test_reset(self):
|
||||||
"""Test that reset() resets all nested connector stats."""
|
"""Test that reset() resets all nested connector stats."""
|
||||||
stats = MultiKVConnectorStats(
|
stats = MultiKVConnectorStats(
|
||||||
|
|||||||
@@ -1676,6 +1676,9 @@ def test_kv_connector_stats_aggregation():
|
|||||||
assert cli_stats["Avg xfer time (ms)"] == 1500.0
|
assert cli_stats["Avg xfer time (ms)"] == 1500.0
|
||||||
assert cli_stats["Avg post time (ms)"] == 1500.0
|
assert cli_stats["Avg post time (ms)"] == 1500.0
|
||||||
assert cli_stats["Avg number of descriptors"] == 1.5
|
assert cli_stats["Avg number of descriptors"] == 1.5
|
||||||
|
# Reduced values must be plain Python scalars so CLI logging renders
|
||||||
|
# them without numpy reprs (eg np.float64(...)).
|
||||||
|
assert all(not isinstance(v, np.generic) for v in cli_stats.values())
|
||||||
|
|
||||||
|
|
||||||
def test_multi_kv_connector_stats_aggregation():
|
def test_multi_kv_connector_stats_aggregation():
|
||||||
|
|||||||
@@ -1075,10 +1075,14 @@ class HF3FSKVConnectorStats(KVConnectorStats):
|
|||||||
"Num save task failed": num_failed_save,
|
"Num save task failed": num_failed_save,
|
||||||
"Num load task success": num_success_load,
|
"Num load task success": num_success_load,
|
||||||
"Num load task failed": num_failed_load,
|
"Num load task failed": num_failed_load,
|
||||||
"Avg save duration (ms)": round(save_duration.mean() * 1e3, 3),
|
"Avg save duration (ms)": round(save_duration.mean().item() * 1e3, 3),
|
||||||
"P90 save duration (ms)": round(np.percentile(save_duration, 90) * 1e3, 3),
|
"P90 save duration (ms)": round(
|
||||||
"Avg load duration (ms)": round(load_duration.mean() * 1e3, 3),
|
np.percentile(save_duration, 90).item() * 1e3, 3
|
||||||
"P90 load duration (ms)": round(np.percentile(load_duration, 90) * 1e3, 3),
|
),
|
||||||
|
"Avg load duration (ms)": round(load_duration.mean().item() * 1e3, 3),
|
||||||
|
"P90 load duration (ms)": round(
|
||||||
|
np.percentile(load_duration, 90).item() * 1e3, 3
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
def is_empty(self) -> bool:
|
def is_empty(self) -> bool:
|
||||||
|
|||||||
@@ -126,16 +126,16 @@ class MooncakeKVConnectorStats(KVConnectorStats):
|
|||||||
avg_mb = total_mb / n
|
avg_mb = total_mb / n
|
||||||
total_time_seconds = xfer_time.sum()
|
total_time_seconds = xfer_time.sum()
|
||||||
throughput_mb_s = (
|
throughput_mb_s = (
|
||||||
total_mb / total_time_seconds if total_time_seconds > 0 else 0.0
|
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"Num successful transfers": n,
|
"Num successful transfers": n,
|
||||||
"Avg xfer time (ms)": round(xfer_time.mean() * 1e3, 3),
|
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||||
"Avg MB per transfer": round(avg_mb, 3),
|
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||||
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
||||||
"Avg number of descriptors": round(descs.mean(), 1),
|
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||||
"Num failed transfers": num_failed_transfers,
|
"Num failed transfers": num_failed_transfers,
|
||||||
"Num failed recvs": num_failed_recvs,
|
"Num failed recvs": num_failed_recvs,
|
||||||
"Num KV expired reqs": num_kv_expired_reqs,
|
"Num KV expired reqs": num_kv_expired_reqs,
|
||||||
|
|||||||
@@ -115,13 +115,13 @@ class NixlKVConnectorStats(KVConnectorStats):
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
"Num successful transfers": n,
|
"Num successful transfers": n,
|
||||||
"Avg xfer time (ms)": round(xfer_time.mean() * 1e3, 3),
|
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||||
"Avg post time (ms)": round(post_time.mean() * 1e3, 3),
|
"Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
|
||||||
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
|
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
|
||||||
"Avg MB per transfer": round(avg_mb, 3),
|
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||||
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
|
||||||
"Avg number of descriptors": round(descs.mean(), 1),
|
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||||
}
|
}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
Reference in New Issue
Block a user