[Bugfix] Fix np.float64 leaking into the KV transfer metrics log (#57068)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-16 18:09:05 +00:00
committed by GitHub
parent 9f9e1dac26
commit fc8132a5e5
7 changed files with 63 additions and 14 deletions
@@ -8,6 +8,7 @@
import os import os
from unittest.mock import MagicMock from unittest.mock import MagicMock
import numpy as np
import pytest import pytest
import torch import torch
@@ -218,6 +219,9 @@ class TestHF3FSKVConnectorStats:
assert result["Num save task success"] == pytest.approx(2.0, rel=0.01) assert result["Num save task success"] == pytest.approx(2.0, rel=0.01)
assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01) assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01)
assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01) assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01)
# Reduced values must be plain Python scalars so CLI logging renders
# them without numpy reprs (eg np.float64(...)).
assert all(not isinstance(v, np.generic) for v in result.values())
def test_clone_and_reset(self, hf3fs_stats): def test_clone_and_reset(self, hf3fs_stats):
"""clone_and_reset() returns a copy with data and resets the original.""" """clone_and_reset() returns a copy with data and resets the original."""
@@ -4,6 +4,8 @@
import threading import threading
from unittest.mock import MagicMock from unittest.mock import MagicMock
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
MooncakeConnector, MooncakeConnector,
MooncakeConnectorWorker, MooncakeConnectorWorker,
@@ -40,6 +42,9 @@ def test_record_transfer_and_reduce():
assert reduced["Num failed transfers"] == 0 assert reduced["Num failed transfers"] == 0
assert reduced["Num failed recvs"] == 0 assert reduced["Num failed recvs"] == 0
assert reduced["Num KV expired reqs"] == 0 assert reduced["Num KV expired reqs"] == 0
# Reduced values must be plain Python scalars so CLI logging renders
# them without numpy reprs (eg np.float64(...)).
assert all(not isinstance(v, np.generic) for v in reduced.values())
def test_record_failures_keeps_stats_non_empty(): def test_record_failures_keeps_stats_non_empty():
@@ -20,7 +20,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
SupportsHMA, SupportsHMA,
supports_hma, supports_hma,
) )
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorLogging,
KVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import ( from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
MultiConnector, MultiConnector,
MultiKVConnectorPromMetrics, MultiKVConnectorPromMetrics,
@@ -817,6 +820,36 @@ class TestMultiConnectorStats:
assert "Num successful transfers" in reduced["NixlConnector"] assert "Num successful transfers" in reduced["NixlConnector"]
assert reduced["NixlConnector"]["Num successful transfers"] == 2 assert reduced["NixlConnector"]["Num successful transfers"] == 2
def test_log_renders_plain_scalars(self):
"""Reduced stats must render as plain numbers in the CLI log, not
numpy reprs (eg np.float64(2.338))."""
stats = MultiKVConnectorStats(
data={
"NixlConnector": NixlKVConnectorStats(
data={
"transfer_duration": [1.0, 2.0],
"post_duration": [0.1, 0.2],
"bytes_transferred": [1024, 2048],
"num_descriptors": [10, 20],
"num_failed_transfers": [],
"num_failed_notifications": [],
}
)
}
)
kv_logging = KVConnectorLogging(kv_transfer_config=None)
kv_logging.transfer_stats_accumulator = stats
log_records: list[tuple] = []
kv_logging.log(log_fn=lambda *args: log_records.append(args))
assert len(log_records) == 1
msg = log_records[0][1]
assert "np.float" not in msg
assert "'Avg xfer time (ms)': 1500.0" in msg
# The accumulator is reset after logging.
assert kv_logging.transfer_stats_accumulator is None
def test_reset(self): def test_reset(self):
"""Test that reset() resets all nested connector stats.""" """Test that reset() resets all nested connector stats."""
stats = MultiKVConnectorStats( stats = MultiKVConnectorStats(
@@ -1676,6 +1676,9 @@ def test_kv_connector_stats_aggregation():
assert cli_stats["Avg xfer time (ms)"] == 1500.0 assert cli_stats["Avg xfer time (ms)"] == 1500.0
assert cli_stats["Avg post time (ms)"] == 1500.0 assert cli_stats["Avg post time (ms)"] == 1500.0
assert cli_stats["Avg number of descriptors"] == 1.5 assert cli_stats["Avg number of descriptors"] == 1.5
# Reduced values must be plain Python scalars so CLI logging renders
# them without numpy reprs (eg np.float64(...)).
assert all(not isinstance(v, np.generic) for v in cli_stats.values())
def test_multi_kv_connector_stats_aggregation(): def test_multi_kv_connector_stats_aggregation():
@@ -1075,10 +1075,14 @@ class HF3FSKVConnectorStats(KVConnectorStats):
"Num save task failed": num_failed_save, "Num save task failed": num_failed_save,
"Num load task success": num_success_load, "Num load task success": num_success_load,
"Num load task failed": num_failed_load, "Num load task failed": num_failed_load,
"Avg save duration (ms)": round(save_duration.mean() * 1e3, 3), "Avg save duration (ms)": round(save_duration.mean().item() * 1e3, 3),
"P90 save duration (ms)": round(np.percentile(save_duration, 90) * 1e3, 3), "P90 save duration (ms)": round(
"Avg load duration (ms)": round(load_duration.mean() * 1e3, 3), np.percentile(save_duration, 90).item() * 1e3, 3
"P90 load duration (ms)": round(np.percentile(load_duration, 90) * 1e3, 3), ),
"Avg load duration (ms)": round(load_duration.mean().item() * 1e3, 3),
"P90 load duration (ms)": round(
np.percentile(load_duration, 90).item() * 1e3, 3
),
} }
def is_empty(self) -> bool: def is_empty(self) -> bool:
@@ -126,16 +126,16 @@ class MooncakeKVConnectorStats(KVConnectorStats):
avg_mb = total_mb / n avg_mb = total_mb / n
total_time_seconds = xfer_time.sum() total_time_seconds = xfer_time.sum()
throughput_mb_s = ( throughput_mb_s = (
total_mb / total_time_seconds if total_time_seconds > 0 else 0.0 (total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
) )
return { return {
"Num successful transfers": n, "Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean() * 1e3, 3), "Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3), "P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb, 3), "Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s, 3), "Throughput (MB/s)": round(throughput_mb_s, 3),
"Avg number of descriptors": round(descs.mean(), 1), "Avg number of descriptors": round(descs.mean().item(), 1),
"Num failed transfers": num_failed_transfers, "Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs, "Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs, "Num KV expired reqs": num_kv_expired_reqs,
@@ -115,13 +115,13 @@ class NixlKVConnectorStats(KVConnectorStats):
return { return {
"Num successful transfers": n, "Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean() * 1e3, 3), "Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3), "P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg post time (ms)": round(post_time.mean() * 1e3, 3), "Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3), "P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb, 3), "Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s, 3), "Throughput (MB/s)": round(throughput_mb_s.item(), 3),
"Avg number of descriptors": round(descs.mean(), 1), "Avg number of descriptors": round(descs.mean().item(), 1),
} }
@property @property