[CI] Sample GPU utilization and memory alongside test timelines (#56541)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Kevin H. Luu
2026-09-12 02:41:54 -07:00
committed by GitHub
co-authored by Codex
parent 756794a9a7
commit 33fa95a0cf
5 changed files with 366 additions and 5 deletions
+43
View File
@@ -0,0 +1,43 @@
# CI timing and GPU samples
The pipeline generator activates these helpers for trusted, traced CI commands.
`ci_otel.sh` records command spans and `ci_pytest.sh` injects the pytest timing
plugin. `ci_gpu.py` runs once per command, outside pytest workers, when
`nvidia-smi` is available. It queries up to 16 container-visible NVIDIA devices
once per second without importing PyTorch or creating a CUDA context.
Set `CI_INFRA_GPU_SAMPLING=0` in the job environment to disable sampling.
Samples contain device UUID, index, name, utilization percent, and used/total
device memory in bytes. They are `ci.gpu.sample` OTLP events inside
`ci.gpu.samples` spans, parented to the command span and authenticated using the
existing job-scoped Buildkite OIDC identity. Batches upload every 30 seconds;
the final batch is spooled for the shell's normal job-end upload. No host agent
or long-lived dashboard token is needed. The dashboard companion change adds
GPU controls alongside jobs, commands, and pytest rows in the Builds timeline.
Device queries and periodic uploads each have a two-second timeout. Three
consecutive query failures stop sampling. Failed periodic uploads are dropped;
memory stays bounded, and the dashboard shows the missing intervals as gaps.
Uploads can delay the next poll. The shell stops and joins the sampler before
flushing traces, allows at most three seconds for shutdown, and preserves the
original command exit status. Samples already uploaded survive an interrupted
job; the last unflushed batch may be lost on forced termination.
These are device readings during each test's time interval, not process-level
kernel attribution. Other processes and overlapping tests can contribute to
the readings. Subsecond tests may have no sample. Missing readings never become
zero utilization. MIG parent-device memory and utilization are omitted because
they cannot be attributed to the job's partition. The initial implementation
covers NVIDIA commands in the existing tracing rollout; CPU jobs and AMD
mirrors do not collect GPU samples.
Run the helper tests without loading vLLM or requiring a GPU:
```bash
PATH="$PWD/.venv/bin:$PATH" .venv/bin/python -m pytest \
.buildkite/scripts/ci-otel/tests/test_ci_otel.py -q
```
The suite uses a fake `nvidia-smi` to exercise the shell lifecycle, device-query
failure handling, disable switch, event encoding, and bounded uploads. Real
hardware sampling overhead still needs a canary run before broad deployment.
+145
View File
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Best-effort, container-visible NVIDIA device samples for command timelines."""
from __future__ import annotations
import csv
import io
import math
import os
import secrets
import signal
import subprocess
import sys
import threading
import time
from ci_otel import Span, export_spans, record_spans
INTERVAL_SECONDS = 1.0
BATCH_SECONDS = 30
MAX_DEVICES = 16
QUERY_TIMEOUT_SECONDS = 2
QUERY = "index,uuid,name,utilization.gpu,memory.used,memory.total,mig.mode.current"
def _number(value: str, scale: int = 1, maximum: int | None = None) -> int | None:
try:
number = float(value)
if not math.isfinite(number) or number < 0:
return None
if maximum is not None and number > maximum:
return None
return round(number * scale)
except ValueError:
return None
def parse_samples(output: str, timestamp_ns: int) -> list[dict]:
events = []
for row in csv.reader(io.StringIO(output)):
if len(row) != 7 or len(events) >= MAX_DEVICES:
continue
index, uuid, name, util, used, total, mig = (value.strip() for value in row)
if not index.isdigit() or not uuid.startswith("GPU-"):
continue
attributes: dict[str, str | int | bool] = {
"gpu.uuid": uuid,
"gpu.index": int(index),
"gpu.name": name,
}
# Parent-device memory on a MIG GPU is not the job's allocated memory.
if mig.lower() == "enabled":
attributes["gpu.status"] = "unsupported_mig"
else:
for key, value in (
("gpu.utilization", _number(util, maximum=100)),
("gpu.memory.used", _number(used, 1024**2)),
("gpu.memory.total", _number(total, 1024**2)),
):
if value is not None:
attributes[key] = value
events.append(
{"time_ns": timestamp_ns, "name": "ci.gpu.sample", "attributes": attributes}
)
return events
def sample_span(events: list[dict]) -> Span:
return Span(
trace_id=os.environ["CI_INFRA_TRACE_ID"],
span_id=secrets.token_hex(8),
parent_span_id=os.environ["CI_INFRA_COMMAND_SPAN_ID"],
name="ci.gpu.samples",
start_ns=events[0]["time_ns"],
end_ns=events[-1]["time_ns"],
attributes={
"ci.span.kind": "gpu-samples",
"gpu.sample.interval_ms": int(INTERVAL_SECONDS * 1000),
"gpu.scope": "container-visible-devices",
},
events=events,
)
def collect(parent_pid: int, stop: threading.Event) -> None:
events: list[dict] = []
batch_started = time.monotonic()
failures = 0
try:
while not stop.is_set() and os.getppid() == parent_pid:
started = time.monotonic()
try:
result = subprocess.run(
[
"nvidia-smi",
f"--query-gpu={QUERY}",
"--format=csv,noheader,nounits",
],
capture_output=True,
text=True,
timeout=QUERY_TIMEOUT_SECONDS,
check=True,
)
if stop.is_set():
break
samples = parse_samples(result.stdout, time.time_ns())
if not samples:
break
events.extend(samples)
failures = 0
except (OSError, subprocess.SubprocessError):
failures += 1
if failures >= 3:
print(
"CI GPU sampling stopped: device query unavailable",
file=sys.stderr,
)
break
if events and time.monotonic() - batch_started >= BATCH_SECONDS:
# Bounded memory and upload time even if the receiver is down.
# Dropped batches appear as gaps, never as zero utilization.
export_spans([sample_span(events)], timeout_seconds=2)
events = []
batch_started = time.monotonic()
stop.wait(max(0, INTERVAL_SECONDS - (time.monotonic() - started)))
finally:
# The shell joins this process before the ordinary job-end spool flush.
if events:
record_spans([sample_span(events)])
def main() -> None:
stop = threading.Event()
signal.signal(signal.SIGTERM, lambda *_: stop.set())
signal.signal(signal.SIGINT, lambda *_: stop.set())
try:
collect(int(sys.argv[1]), stop)
except Exception:
print("CI GPU sampling skipped", file=sys.stderr)
if __name__ == "__main__":
main()
+25 -3
View File
@@ -14,12 +14,13 @@ import sys
import time
import urllib.request
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field
from pathlib import Path
ENDPOINT = os.getenv("CI_INFRA_OTEL_ENDPOINT", "https://ci.vllm.ai/api/otel/v1/traces")
AUDIENCE = os.getenv("CI_INFRA_OTEL_AUDIENCE", "https://ci.vllm.ai/api/otel")
MAX_BATCH_SIZE = 2_000
MAX_BATCH_BYTES = 1024 * 1024
TEST_SPOOL_BATCH_SIZE = 100
@@ -56,6 +57,7 @@ class Span:
end_ns: int
attributes: dict[str, str | int | bool]
status_code: int = 1
events: list[dict] = field(default_factory=list)
def _varint(value: int) -> bytes:
@@ -137,6 +139,12 @@ def _encode_span(span: Span) -> bytes:
)
for name, value in attributes.items():
encoded += _bytes_field(9, _attribute(name, value))
for event in span.events:
encoded_event = _fixed64_field(1, int(event["time_ns"]))
encoded_event += _string_field(2, event["name"])
for name, value in event["attributes"].items():
encoded_event += _bytes_field(3, _attribute(name, value))
encoded += _bytes_field(11, encoded_event)
return encoded + _bytes_field(15, _varint_field(3, span.status_code))
@@ -482,10 +490,24 @@ def export_spans(spans: list[Span], timeout_seconds: float = 3.0) -> bool:
try:
deadline = time.monotonic() + max(timeout_seconds, 0.1)
token = _oidc_token(deadline)
for offset in range(0, len(spans), MAX_BATCH_SIZE):
batches: list[list[Span]] = []
batch: list[Span] = []
size = 0
for span in spans:
span_size = len(_encode_span(span))
if batch and (
size + span_size > MAX_BATCH_BYTES or len(batch) >= MAX_BATCH_SIZE
):
batches.append(batch)
batch, size = [], 0
batch.append(span)
size += span_size
if batch:
batches.append(batch)
for batch in batches:
request = urllib.request.Request(
ENDPOINT,
data=encode_request(spans[offset : offset + MAX_BATCH_SIZE]),
data=encode_request(batch),
method="POST",
headers={
"Authorization": f"Bearer {token}",
+19
View File
@@ -82,6 +82,13 @@ ci_otel_start() {
CI_INFRA_COMMAND_SPAN_ID="${_CI_INFRA_OTEL_COMMAND_SPAN_ID}"
export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID
_CI_INFRA_OTEL_ACTIVE=1
_CI_INFRA_GPU_PID=""
if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] &&
[ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] &&
command -v nvidia-smi >/dev/null 2>&1; then
"${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" </dev/null &
_CI_INFRA_GPU_PID=$!
fi
}
ci_otel_finish() {
@@ -93,6 +100,18 @@ ci_otel_finish() {
"${_CI_INFRA_OTEL_PARENT_SPAN_ID}" "${_CI_INFRA_OTEL_START_NS}" \
"${_CI_INFRA_OTEL_COMMAND_INDEX}" "${_CI_INFRA_OTEL_COMMAND_STATUS}" \
"${_CI_INFRA_OTEL_COMMAND_LABEL}" || :
if [ -n "${_CI_INFRA_GPU_PID:-}" ]; then
kill -TERM "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
_CI_INFRA_GPU_WAIT=0
while kill -0 "${_CI_INFRA_GPU_PID}" 2>/dev/null &&
[ "${_CI_INFRA_GPU_WAIT}" -lt 30 ]; do
sleep 0.1
_CI_INFRA_GPU_WAIT=$((_CI_INFRA_GPU_WAIT + 1))
done
kill -KILL "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
wait "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
_CI_INFRA_GPU_PID=""
fi
CI_INFRA_TRACE_ID=""
CI_INFRA_COMMAND_SPAN_ID=""
export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID
@@ -8,6 +8,7 @@ import os
import shlex
import subprocess
import sys
import threading
import time
from pathlib import Path
@@ -16,6 +17,7 @@ import pytest
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SCRIPTS_DIR))
import ci_gpu # noqa: E402
import ci_otel # noqa: E402
from ci_otel import Span # noqa: E402
@@ -251,9 +253,15 @@ def test_export_failure_is_soft_and_bounded(monkeypatch):
assert time.monotonic() - started < 0.5
def test_export_batches_with_one_oidc_token(monkeypatch):
@pytest.mark.parametrize("byte_limited", [False, True])
def test_export_batches_with_one_oidc_token(monkeypatch, byte_limited):
monkeypatch.setenv("BUILDKITE", "true")
monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2)
if byte_limited:
monkeypatch.setattr(
ci_otel, "MAX_BATCH_BYTES", len(ci_otel._encode_span(_span())) * 2
)
else:
monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2)
token_calls = []
requests = []
@@ -786,3 +794,127 @@ def test_pytest_plugin_failure_cannot_fail_pytest(tmp_path):
assert result.returncode == 0, result.stderr
assert "1 passed" in result.stdout
def test_gpu_samples_preserve_zero_and_omit_unsupported_metrics():
events = ci_gpu.parse_samples(
"0, GPU-a, H200, 0, 1024, 2048, Disabled\n"
"1, GPU-b, H200, [N/A], [N/A], 2048, Disabled\n"
"2, GPU-c, H200, 50, 1024, 2048, Enabled\n"
"3, GPU-d, H200, NaN, -1, inf, Disabled\n",
1234,
)
assert len(events) == 4
assert events[0]["attributes"]["gpu.utilization"] == 0
assert events[0]["attributes"]["gpu.memory.used"] == 1024**3
for event in events[1:]:
assert "gpu.utilization" not in event["attributes"]
assert "gpu.memory.used" not in event["attributes"]
assert events[2]["attributes"]["gpu.status"] == "unsupported_mig"
assert "gpu.memory.total" not in events[2]["attributes"]
def test_gpu_batches_keep_command_identity_and_survive_spool_roundtrip(
monkeypatch, tmp_path
):
monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16)
monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8)
monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path))
span = ci_gpu.sample_span(
ci_gpu.parse_samples("0, GPU-a, H200, 90, 1, 2, Disabled", 1234)
)
assert span.parent_span_id == "02" * 8
assert span.trace_id == "01" * 16
assert ci_otel.record_spans([span])
assert ci_otel.load_spans() == [span]
assert b"ci.gpu.sample" in ci_otel.encode_request([span])
def test_gpu_query_failures_are_bounded_and_never_synthesize_idle_samples(monkeypatch):
calls = []
def query(*args, **kwargs):
calls.append(kwargs["timeout"])
raise subprocess.TimeoutExpired("nvidia-smi", kwargs["timeout"])
monkeypatch.setattr(ci_gpu.subprocess, "run", query)
monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0)
monkeypatch.setattr(
ci_gpu, "record_spans", lambda _: pytest.fail("fabricated sample")
)
ci_gpu.collect(os.getppid(), threading.Event())
assert calls == [2, 2, 2]
def test_gpu_periodic_batches_are_bounded_even_when_upload_fails(monkeypatch):
monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16)
monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8)
monkeypatch.setattr(ci_gpu, "BATCH_SECONDS", 0)
monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0)
monkeypatch.setattr(
ci_gpu.subprocess,
"run",
lambda *a, **k: subprocess.CompletedProcess(
a, 0, "0, GPU-a, H200, 50, 1, 2, Disabled"
),
)
stop = threading.Event()
batches = []
def upload(spans, timeout_seconds):
batches.append(spans)
assert timeout_seconds == 2
if len(batches) == 3:
stop.set()
return False
monkeypatch.setattr(ci_gpu, "export_spans", upload)
ci_gpu.collect(os.getppid(), stop)
assert [len(batch[0].events) for batch in batches] == [1, 1, 1]
@pytest.mark.parametrize("sampling", ["1", "0"])
def test_gpu_sampler_stops_with_command_and_preserves_failure_status(
tmp_path, sampling
):
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
smi = bin_dir / "nvidia-smi"
smi.write_text('#!/bin/sh\necho "0, GPU-a, H200, 70, 1024, 2048, Disabled"\n')
smi.chmod(0o755)
# Keep the spool for inspection, and avoid any network upload.
shell = (
f'. "{SCRIPTS_DIR / "ci_otel.sh"}"\n'
"ci_otel_run 1 workload sh -c 'sleep 1.3; exit 7'\n"
"exit $?"
)
result = subprocess.run(
["/bin/sh", "-c", shell],
capture_output=True,
text=True,
timeout=8,
env={
**os.environ,
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"BUILDKITE": "false",
"CI_INFRA_GPU_SAMPLING": sampling,
"CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),
"CI_INFRA_OTEL_RUNTIME_DIR": str(tmp_path / "runtime"),
"CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path / "spans"),
},
)
assert result.returncode == 7, result.stderr
records = [
json.loads(line)
for path in (tmp_path / "spans").glob("spans-*.jsonl")
for line in path.read_text().splitlines()
]
command = next(record for record in records if record["name"] == "ci.command")
batches = [record for record in records if record["name"] == "ci.gpu.samples"]
assert bool(batches) == (sampling == "1")
if batches:
assert batches[0]["parent_span_id"] == command["span_id"]
assert all(
command["start_ns"] <= event["time_ns"] <= command["end_ns"]
for event in batches[0]["events"]
)