[CI] Sample GPU utilization and memory alongside test timelines (#56541)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
# CI timing and GPU samples
|
||||
|
||||
The pipeline generator activates these helpers for trusted, traced CI commands.
|
||||
`ci_otel.sh` records command spans and `ci_pytest.sh` injects the pytest timing
|
||||
plugin. `ci_gpu.py` runs once per command, outside pytest workers, when
|
||||
`nvidia-smi` is available. It queries up to 16 container-visible NVIDIA devices
|
||||
once per second without importing PyTorch or creating a CUDA context.
|
||||
Set `CI_INFRA_GPU_SAMPLING=0` in the job environment to disable sampling.
|
||||
|
||||
Samples contain device UUID, index, name, utilization percent, and used/total
|
||||
device memory in bytes. They are `ci.gpu.sample` OTLP events inside
|
||||
`ci.gpu.samples` spans, parented to the command span and authenticated using the
|
||||
existing job-scoped Buildkite OIDC identity. Batches upload every 30 seconds;
|
||||
the final batch is spooled for the shell's normal job-end upload. No host agent
|
||||
or long-lived dashboard token is needed. The dashboard companion change adds
|
||||
GPU controls alongside jobs, commands, and pytest rows in the Builds timeline.
|
||||
|
||||
Device queries and periodic uploads each have a two-second timeout. Three
|
||||
consecutive query failures stop sampling. Failed periodic uploads are dropped;
|
||||
memory stays bounded, and the dashboard shows the missing intervals as gaps.
|
||||
Uploads can delay the next poll. The shell stops and joins the sampler before
|
||||
flushing traces, allows at most three seconds for shutdown, and preserves the
|
||||
original command exit status. Samples already uploaded survive an interrupted
|
||||
job; the last unflushed batch may be lost on forced termination.
|
||||
|
||||
These are device readings during each test's time interval, not process-level
|
||||
kernel attribution. Other processes and overlapping tests can contribute to
|
||||
the readings. Subsecond tests may have no sample. Missing readings never become
|
||||
zero utilization. MIG parent-device memory and utilization are omitted because
|
||||
they cannot be attributed to the job's partition. The initial implementation
|
||||
covers NVIDIA commands in the existing tracing rollout; CPU jobs and AMD
|
||||
mirrors do not collect GPU samples.
|
||||
|
||||
Run the helper tests without loading vLLM or requiring a GPU:
|
||||
|
||||
```bash
|
||||
PATH="$PWD/.venv/bin:$PATH" .venv/bin/python -m pytest \
|
||||
.buildkite/scripts/ci-otel/tests/test_ci_otel.py -q
|
||||
```
|
||||
|
||||
The suite uses a fake `nvidia-smi` to exercise the shell lifecycle, device-query
|
||||
failure handling, disable switch, event encoding, and bounded uploads. Real
|
||||
hardware sampling overhead still needs a canary run before broad deployment.
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Best-effort, container-visible NVIDIA device samples for command timelines."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from ci_otel import Span, export_spans, record_spans
|
||||
|
||||
INTERVAL_SECONDS = 1.0
|
||||
BATCH_SECONDS = 30
|
||||
MAX_DEVICES = 16
|
||||
QUERY_TIMEOUT_SECONDS = 2
|
||||
QUERY = "index,uuid,name,utilization.gpu,memory.used,memory.total,mig.mode.current"
|
||||
|
||||
|
||||
def _number(value: str, scale: int = 1, maximum: int | None = None) -> int | None:
|
||||
try:
|
||||
number = float(value)
|
||||
if not math.isfinite(number) or number < 0:
|
||||
return None
|
||||
if maximum is not None and number > maximum:
|
||||
return None
|
||||
return round(number * scale)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_samples(output: str, timestamp_ns: int) -> list[dict]:
|
||||
events = []
|
||||
for row in csv.reader(io.StringIO(output)):
|
||||
if len(row) != 7 or len(events) >= MAX_DEVICES:
|
||||
continue
|
||||
index, uuid, name, util, used, total, mig = (value.strip() for value in row)
|
||||
if not index.isdigit() or not uuid.startswith("GPU-"):
|
||||
continue
|
||||
attributes: dict[str, str | int | bool] = {
|
||||
"gpu.uuid": uuid,
|
||||
"gpu.index": int(index),
|
||||
"gpu.name": name,
|
||||
}
|
||||
# Parent-device memory on a MIG GPU is not the job's allocated memory.
|
||||
if mig.lower() == "enabled":
|
||||
attributes["gpu.status"] = "unsupported_mig"
|
||||
else:
|
||||
for key, value in (
|
||||
("gpu.utilization", _number(util, maximum=100)),
|
||||
("gpu.memory.used", _number(used, 1024**2)),
|
||||
("gpu.memory.total", _number(total, 1024**2)),
|
||||
):
|
||||
if value is not None:
|
||||
attributes[key] = value
|
||||
events.append(
|
||||
{"time_ns": timestamp_ns, "name": "ci.gpu.sample", "attributes": attributes}
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
def sample_span(events: list[dict]) -> Span:
|
||||
return Span(
|
||||
trace_id=os.environ["CI_INFRA_TRACE_ID"],
|
||||
span_id=secrets.token_hex(8),
|
||||
parent_span_id=os.environ["CI_INFRA_COMMAND_SPAN_ID"],
|
||||
name="ci.gpu.samples",
|
||||
start_ns=events[0]["time_ns"],
|
||||
end_ns=events[-1]["time_ns"],
|
||||
attributes={
|
||||
"ci.span.kind": "gpu-samples",
|
||||
"gpu.sample.interval_ms": int(INTERVAL_SECONDS * 1000),
|
||||
"gpu.scope": "container-visible-devices",
|
||||
},
|
||||
events=events,
|
||||
)
|
||||
|
||||
|
||||
def collect(parent_pid: int, stop: threading.Event) -> None:
|
||||
events: list[dict] = []
|
||||
batch_started = time.monotonic()
|
||||
failures = 0
|
||||
try:
|
||||
while not stop.is_set() and os.getppid() == parent_pid:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"nvidia-smi",
|
||||
f"--query-gpu={QUERY}",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=QUERY_TIMEOUT_SECONDS,
|
||||
check=True,
|
||||
)
|
||||
if stop.is_set():
|
||||
break
|
||||
samples = parse_samples(result.stdout, time.time_ns())
|
||||
if not samples:
|
||||
break
|
||||
events.extend(samples)
|
||||
failures = 0
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
failures += 1
|
||||
if failures >= 3:
|
||||
print(
|
||||
"CI GPU sampling stopped: device query unavailable",
|
||||
file=sys.stderr,
|
||||
)
|
||||
break
|
||||
if events and time.monotonic() - batch_started >= BATCH_SECONDS:
|
||||
# Bounded memory and upload time even if the receiver is down.
|
||||
# Dropped batches appear as gaps, never as zero utilization.
|
||||
export_spans([sample_span(events)], timeout_seconds=2)
|
||||
events = []
|
||||
batch_started = time.monotonic()
|
||||
stop.wait(max(0, INTERVAL_SECONDS - (time.monotonic() - started)))
|
||||
finally:
|
||||
# The shell joins this process before the ordinary job-end spool flush.
|
||||
if events:
|
||||
record_spans([sample_span(events)])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
stop = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: stop.set())
|
||||
try:
|
||||
collect(int(sys.argv[1]), stop)
|
||||
except Exception:
|
||||
print("CI GPU sampling skipped", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,12 +14,13 @@ import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
ENDPOINT = os.getenv("CI_INFRA_OTEL_ENDPOINT", "https://ci.vllm.ai/api/otel/v1/traces")
|
||||
AUDIENCE = os.getenv("CI_INFRA_OTEL_AUDIENCE", "https://ci.vllm.ai/api/otel")
|
||||
MAX_BATCH_SIZE = 2_000
|
||||
MAX_BATCH_BYTES = 1024 * 1024
|
||||
TEST_SPOOL_BATCH_SIZE = 100
|
||||
|
||||
|
||||
@@ -56,6 +57,7 @@ class Span:
|
||||
end_ns: int
|
||||
attributes: dict[str, str | int | bool]
|
||||
status_code: int = 1
|
||||
events: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
def _varint(value: int) -> bytes:
|
||||
@@ -137,6 +139,12 @@ def _encode_span(span: Span) -> bytes:
|
||||
)
|
||||
for name, value in attributes.items():
|
||||
encoded += _bytes_field(9, _attribute(name, value))
|
||||
for event in span.events:
|
||||
encoded_event = _fixed64_field(1, int(event["time_ns"]))
|
||||
encoded_event += _string_field(2, event["name"])
|
||||
for name, value in event["attributes"].items():
|
||||
encoded_event += _bytes_field(3, _attribute(name, value))
|
||||
encoded += _bytes_field(11, encoded_event)
|
||||
return encoded + _bytes_field(15, _varint_field(3, span.status_code))
|
||||
|
||||
|
||||
@@ -482,10 +490,24 @@ def export_spans(spans: list[Span], timeout_seconds: float = 3.0) -> bool:
|
||||
try:
|
||||
deadline = time.monotonic() + max(timeout_seconds, 0.1)
|
||||
token = _oidc_token(deadline)
|
||||
for offset in range(0, len(spans), MAX_BATCH_SIZE):
|
||||
batches: list[list[Span]] = []
|
||||
batch: list[Span] = []
|
||||
size = 0
|
||||
for span in spans:
|
||||
span_size = len(_encode_span(span))
|
||||
if batch and (
|
||||
size + span_size > MAX_BATCH_BYTES or len(batch) >= MAX_BATCH_SIZE
|
||||
):
|
||||
batches.append(batch)
|
||||
batch, size = [], 0
|
||||
batch.append(span)
|
||||
size += span_size
|
||||
if batch:
|
||||
batches.append(batch)
|
||||
for batch in batches:
|
||||
request = urllib.request.Request(
|
||||
ENDPOINT,
|
||||
data=encode_request(spans[offset : offset + MAX_BATCH_SIZE]),
|
||||
data=encode_request(batch),
|
||||
method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
|
||||
@@ -82,6 +82,13 @@ ci_otel_start() {
|
||||
CI_INFRA_COMMAND_SPAN_ID="${_CI_INFRA_OTEL_COMMAND_SPAN_ID}"
|
||||
export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID
|
||||
_CI_INFRA_OTEL_ACTIVE=1
|
||||
_CI_INFRA_GPU_PID=""
|
||||
if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] &&
|
||||
[ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] &&
|
||||
command -v nvidia-smi >/dev/null 2>&1; then
|
||||
"${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" </dev/null &
|
||||
_CI_INFRA_GPU_PID=$!
|
||||
fi
|
||||
}
|
||||
|
||||
ci_otel_finish() {
|
||||
@@ -93,6 +100,18 @@ ci_otel_finish() {
|
||||
"${_CI_INFRA_OTEL_PARENT_SPAN_ID}" "${_CI_INFRA_OTEL_START_NS}" \
|
||||
"${_CI_INFRA_OTEL_COMMAND_INDEX}" "${_CI_INFRA_OTEL_COMMAND_STATUS}" \
|
||||
"${_CI_INFRA_OTEL_COMMAND_LABEL}" || :
|
||||
if [ -n "${_CI_INFRA_GPU_PID:-}" ]; then
|
||||
kill -TERM "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
|
||||
_CI_INFRA_GPU_WAIT=0
|
||||
while kill -0 "${_CI_INFRA_GPU_PID}" 2>/dev/null &&
|
||||
[ "${_CI_INFRA_GPU_WAIT}" -lt 30 ]; do
|
||||
sleep 0.1
|
||||
_CI_INFRA_GPU_WAIT=$((_CI_INFRA_GPU_WAIT + 1))
|
||||
done
|
||||
kill -KILL "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
|
||||
wait "${_CI_INFRA_GPU_PID}" 2>/dev/null || :
|
||||
_CI_INFRA_GPU_PID=""
|
||||
fi
|
||||
CI_INFRA_TRACE_ID=""
|
||||
CI_INFRA_COMMAND_SPAN_ID=""
|
||||
export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID
|
||||
|
||||
@@ -8,6 +8,7 @@ import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@@ -16,6 +17,7 @@ import pytest
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import ci_gpu # noqa: E402
|
||||
import ci_otel # noqa: E402
|
||||
from ci_otel import Span # noqa: E402
|
||||
|
||||
@@ -251,9 +253,15 @@ def test_export_failure_is_soft_and_bounded(monkeypatch):
|
||||
assert time.monotonic() - started < 0.5
|
||||
|
||||
|
||||
def test_export_batches_with_one_oidc_token(monkeypatch):
|
||||
@pytest.mark.parametrize("byte_limited", [False, True])
|
||||
def test_export_batches_with_one_oidc_token(monkeypatch, byte_limited):
|
||||
monkeypatch.setenv("BUILDKITE", "true")
|
||||
monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2)
|
||||
if byte_limited:
|
||||
monkeypatch.setattr(
|
||||
ci_otel, "MAX_BATCH_BYTES", len(ci_otel._encode_span(_span())) * 2
|
||||
)
|
||||
else:
|
||||
monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2)
|
||||
token_calls = []
|
||||
requests = []
|
||||
|
||||
@@ -786,3 +794,127 @@ def test_pytest_plugin_failure_cannot_fail_pytest(tmp_path):
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "1 passed" in result.stdout
|
||||
|
||||
|
||||
def test_gpu_samples_preserve_zero_and_omit_unsupported_metrics():
|
||||
events = ci_gpu.parse_samples(
|
||||
"0, GPU-a, H200, 0, 1024, 2048, Disabled\n"
|
||||
"1, GPU-b, H200, [N/A], [N/A], 2048, Disabled\n"
|
||||
"2, GPU-c, H200, 50, 1024, 2048, Enabled\n"
|
||||
"3, GPU-d, H200, NaN, -1, inf, Disabled\n",
|
||||
1234,
|
||||
)
|
||||
assert len(events) == 4
|
||||
assert events[0]["attributes"]["gpu.utilization"] == 0
|
||||
assert events[0]["attributes"]["gpu.memory.used"] == 1024**3
|
||||
for event in events[1:]:
|
||||
assert "gpu.utilization" not in event["attributes"]
|
||||
assert "gpu.memory.used" not in event["attributes"]
|
||||
assert events[2]["attributes"]["gpu.status"] == "unsupported_mig"
|
||||
assert "gpu.memory.total" not in events[2]["attributes"]
|
||||
|
||||
|
||||
def test_gpu_batches_keep_command_identity_and_survive_spool_roundtrip(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16)
|
||||
monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8)
|
||||
monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path))
|
||||
span = ci_gpu.sample_span(
|
||||
ci_gpu.parse_samples("0, GPU-a, H200, 90, 1, 2, Disabled", 1234)
|
||||
)
|
||||
assert span.parent_span_id == "02" * 8
|
||||
assert span.trace_id == "01" * 16
|
||||
assert ci_otel.record_spans([span])
|
||||
assert ci_otel.load_spans() == [span]
|
||||
assert b"ci.gpu.sample" in ci_otel.encode_request([span])
|
||||
|
||||
|
||||
def test_gpu_query_failures_are_bounded_and_never_synthesize_idle_samples(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def query(*args, **kwargs):
|
||||
calls.append(kwargs["timeout"])
|
||||
raise subprocess.TimeoutExpired("nvidia-smi", kwargs["timeout"])
|
||||
|
||||
monkeypatch.setattr(ci_gpu.subprocess, "run", query)
|
||||
monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0)
|
||||
monkeypatch.setattr(
|
||||
ci_gpu, "record_spans", lambda _: pytest.fail("fabricated sample")
|
||||
)
|
||||
ci_gpu.collect(os.getppid(), threading.Event())
|
||||
assert calls == [2, 2, 2]
|
||||
|
||||
|
||||
def test_gpu_periodic_batches_are_bounded_even_when_upload_fails(monkeypatch):
|
||||
monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16)
|
||||
monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8)
|
||||
monkeypatch.setattr(ci_gpu, "BATCH_SECONDS", 0)
|
||||
monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0)
|
||||
monkeypatch.setattr(
|
||||
ci_gpu.subprocess,
|
||||
"run",
|
||||
lambda *a, **k: subprocess.CompletedProcess(
|
||||
a, 0, "0, GPU-a, H200, 50, 1, 2, Disabled"
|
||||
),
|
||||
)
|
||||
stop = threading.Event()
|
||||
batches = []
|
||||
|
||||
def upload(spans, timeout_seconds):
|
||||
batches.append(spans)
|
||||
assert timeout_seconds == 2
|
||||
if len(batches) == 3:
|
||||
stop.set()
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(ci_gpu, "export_spans", upload)
|
||||
ci_gpu.collect(os.getppid(), stop)
|
||||
assert [len(batch[0].events) for batch in batches] == [1, 1, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sampling", ["1", "0"])
|
||||
def test_gpu_sampler_stops_with_command_and_preserves_failure_status(
|
||||
tmp_path, sampling
|
||||
):
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
smi = bin_dir / "nvidia-smi"
|
||||
smi.write_text('#!/bin/sh\necho "0, GPU-a, H200, 70, 1024, 2048, Disabled"\n')
|
||||
smi.chmod(0o755)
|
||||
# Keep the spool for inspection, and avoid any network upload.
|
||||
shell = (
|
||||
f'. "{SCRIPTS_DIR / "ci_otel.sh"}"\n'
|
||||
"ci_otel_run 1 workload sh -c 'sleep 1.3; exit 7'\n"
|
||||
"exit $?"
|
||||
)
|
||||
result = subprocess.run(
|
||||
["/bin/sh", "-c", shell],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=8,
|
||||
env={
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"BUILDKITE": "false",
|
||||
"CI_INFRA_GPU_SAMPLING": sampling,
|
||||
"CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),
|
||||
"CI_INFRA_OTEL_RUNTIME_DIR": str(tmp_path / "runtime"),
|
||||
"CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path / "spans"),
|
||||
},
|
||||
)
|
||||
assert result.returncode == 7, result.stderr
|
||||
records = [
|
||||
json.loads(line)
|
||||
for path in (tmp_path / "spans").glob("spans-*.jsonl")
|
||||
for line in path.read_text().splitlines()
|
||||
]
|
||||
command = next(record for record in records if record["name"] == "ci.command")
|
||||
batches = [record for record in records if record["name"] == "ci.gpu.samples"]
|
||||
assert bool(batches) == (sampling == "1")
|
||||
if batches:
|
||||
assert batches[0]["parent_span_id"] == command["span_id"]
|
||||
assert all(
|
||||
command["start_ns"] <= event["time_ns"] <= command["end_ns"]
|
||||
for event in batches[0]["events"]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user