From 33fa95a0cf9ed6bf384a0a51667a01a4f0bb02d5 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Sat, 12 Sep 2026 02:41:54 -0700 Subject: [PATCH] [CI] Sample GPU utilization and memory alongside test timelines (#56541) Signed-off-by: khluu Co-authored-by: Codex --- .buildkite/scripts/ci-otel/README.md | 43 ++++++ .buildkite/scripts/ci-otel/ci_gpu.py | 145 ++++++++++++++++++ .buildkite/scripts/ci-otel/ci_otel.py | 28 +++- .buildkite/scripts/ci-otel/ci_otel.sh | 19 +++ .../scripts/ci-otel/tests/test_ci_otel.py | 136 +++++++++++++++- 5 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 .buildkite/scripts/ci-otel/README.md create mode 100644 .buildkite/scripts/ci-otel/ci_gpu.py diff --git a/.buildkite/scripts/ci-otel/README.md b/.buildkite/scripts/ci-otel/README.md new file mode 100644 index 0000000000..3ad552b5db --- /dev/null +++ b/.buildkite/scripts/ci-otel/README.md @@ -0,0 +1,43 @@ +# CI timing and GPU samples + +The pipeline generator activates these helpers for trusted, traced CI commands. +`ci_otel.sh` records command spans and `ci_pytest.sh` injects the pytest timing +plugin. `ci_gpu.py` runs once per command, outside pytest workers, when +`nvidia-smi` is available. It queries up to 16 container-visible NVIDIA devices +once per second without importing PyTorch or creating a CUDA context. +Set `CI_INFRA_GPU_SAMPLING=0` in the job environment to disable sampling. + +Samples contain device UUID, index, name, utilization percent, and used/total +device memory in bytes. They are `ci.gpu.sample` OTLP events inside +`ci.gpu.samples` spans, parented to the command span and authenticated using the +existing job-scoped Buildkite OIDC identity. Batches upload every 30 seconds; +the final batch is spooled for the shell's normal job-end upload. No host agent +or long-lived dashboard token is needed. The dashboard companion change adds +GPU controls alongside jobs, commands, and pytest rows in the Builds timeline. + +Device queries and periodic uploads each have a two-second timeout. Three +consecutive query failures stop sampling. Failed periodic uploads are dropped; +memory stays bounded, and the dashboard shows the missing intervals as gaps. +Uploads can delay the next poll. The shell stops and joins the sampler before +flushing traces, allows at most three seconds for shutdown, and preserves the +original command exit status. Samples already uploaded survive an interrupted +job; the last unflushed batch may be lost on forced termination. + +These are device readings during each test's time interval, not process-level +kernel attribution. Other processes and overlapping tests can contribute to +the readings. Subsecond tests may have no sample. Missing readings never become +zero utilization. MIG parent-device memory and utilization are omitted because +they cannot be attributed to the job's partition. The initial implementation +covers NVIDIA commands in the existing tracing rollout; CPU jobs and AMD +mirrors do not collect GPU samples. + +Run the helper tests without loading vLLM or requiring a GPU: + +```bash +PATH="$PWD/.venv/bin:$PATH" .venv/bin/python -m pytest \ + .buildkite/scripts/ci-otel/tests/test_ci_otel.py -q +``` + +The suite uses a fake `nvidia-smi` to exercise the shell lifecycle, device-query +failure handling, disable switch, event encoding, and bounded uploads. Real +hardware sampling overhead still needs a canary run before broad deployment. diff --git a/.buildkite/scripts/ci-otel/ci_gpu.py b/.buildkite/scripts/ci-otel/ci_gpu.py new file mode 100644 index 0000000000..304d738b88 --- /dev/null +++ b/.buildkite/scripts/ci-otel/ci_gpu.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Best-effort, container-visible NVIDIA device samples for command timelines.""" + +from __future__ import annotations + +import csv +import io +import math +import os +import secrets +import signal +import subprocess +import sys +import threading +import time + +from ci_otel import Span, export_spans, record_spans + +INTERVAL_SECONDS = 1.0 +BATCH_SECONDS = 30 +MAX_DEVICES = 16 +QUERY_TIMEOUT_SECONDS = 2 +QUERY = "index,uuid,name,utilization.gpu,memory.used,memory.total,mig.mode.current" + + +def _number(value: str, scale: int = 1, maximum: int | None = None) -> int | None: + try: + number = float(value) + if not math.isfinite(number) or number < 0: + return None + if maximum is not None and number > maximum: + return None + return round(number * scale) + except ValueError: + return None + + +def parse_samples(output: str, timestamp_ns: int) -> list[dict]: + events = [] + for row in csv.reader(io.StringIO(output)): + if len(row) != 7 or len(events) >= MAX_DEVICES: + continue + index, uuid, name, util, used, total, mig = (value.strip() for value in row) + if not index.isdigit() or not uuid.startswith("GPU-"): + continue + attributes: dict[str, str | int | bool] = { + "gpu.uuid": uuid, + "gpu.index": int(index), + "gpu.name": name, + } + # Parent-device memory on a MIG GPU is not the job's allocated memory. + if mig.lower() == "enabled": + attributes["gpu.status"] = "unsupported_mig" + else: + for key, value in ( + ("gpu.utilization", _number(util, maximum=100)), + ("gpu.memory.used", _number(used, 1024**2)), + ("gpu.memory.total", _number(total, 1024**2)), + ): + if value is not None: + attributes[key] = value + events.append( + {"time_ns": timestamp_ns, "name": "ci.gpu.sample", "attributes": attributes} + ) + return events + + +def sample_span(events: list[dict]) -> Span: + return Span( + trace_id=os.environ["CI_INFRA_TRACE_ID"], + span_id=secrets.token_hex(8), + parent_span_id=os.environ["CI_INFRA_COMMAND_SPAN_ID"], + name="ci.gpu.samples", + start_ns=events[0]["time_ns"], + end_ns=events[-1]["time_ns"], + attributes={ + "ci.span.kind": "gpu-samples", + "gpu.sample.interval_ms": int(INTERVAL_SECONDS * 1000), + "gpu.scope": "container-visible-devices", + }, + events=events, + ) + + +def collect(parent_pid: int, stop: threading.Event) -> None: + events: list[dict] = [] + batch_started = time.monotonic() + failures = 0 + try: + while not stop.is_set() and os.getppid() == parent_pid: + started = time.monotonic() + try: + result = subprocess.run( + [ + "nvidia-smi", + f"--query-gpu={QUERY}", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=QUERY_TIMEOUT_SECONDS, + check=True, + ) + if stop.is_set(): + break + samples = parse_samples(result.stdout, time.time_ns()) + if not samples: + break + events.extend(samples) + failures = 0 + except (OSError, subprocess.SubprocessError): + failures += 1 + if failures >= 3: + print( + "CI GPU sampling stopped: device query unavailable", + file=sys.stderr, + ) + break + if events and time.monotonic() - batch_started >= BATCH_SECONDS: + # Bounded memory and upload time even if the receiver is down. + # Dropped batches appear as gaps, never as zero utilization. + export_spans([sample_span(events)], timeout_seconds=2) + events = [] + batch_started = time.monotonic() + stop.wait(max(0, INTERVAL_SECONDS - (time.monotonic() - started))) + finally: + # The shell joins this process before the ordinary job-end spool flush. + if events: + record_spans([sample_span(events)]) + + +def main() -> None: + stop = threading.Event() + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + signal.signal(signal.SIGINT, lambda *_: stop.set()) + try: + collect(int(sys.argv[1]), stop) + except Exception: + print("CI GPU sampling skipped", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/.buildkite/scripts/ci-otel/ci_otel.py b/.buildkite/scripts/ci-otel/ci_otel.py index 0d25594bec..cec3f41309 100644 --- a/.buildkite/scripts/ci-otel/ci_otel.py +++ b/.buildkite/scripts/ci-otel/ci_otel.py @@ -14,12 +14,13 @@ import sys import time import urllib.request from collections.abc import Iterable -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from pathlib import Path ENDPOINT = os.getenv("CI_INFRA_OTEL_ENDPOINT", "https://ci.vllm.ai/api/otel/v1/traces") AUDIENCE = os.getenv("CI_INFRA_OTEL_AUDIENCE", "https://ci.vllm.ai/api/otel") MAX_BATCH_SIZE = 2_000 +MAX_BATCH_BYTES = 1024 * 1024 TEST_SPOOL_BATCH_SIZE = 100 @@ -56,6 +57,7 @@ class Span: end_ns: int attributes: dict[str, str | int | bool] status_code: int = 1 + events: list[dict] = field(default_factory=list) def _varint(value: int) -> bytes: @@ -137,6 +139,12 @@ def _encode_span(span: Span) -> bytes: ) for name, value in attributes.items(): encoded += _bytes_field(9, _attribute(name, value)) + for event in span.events: + encoded_event = _fixed64_field(1, int(event["time_ns"])) + encoded_event += _string_field(2, event["name"]) + for name, value in event["attributes"].items(): + encoded_event += _bytes_field(3, _attribute(name, value)) + encoded += _bytes_field(11, encoded_event) return encoded + _bytes_field(15, _varint_field(3, span.status_code)) @@ -482,10 +490,24 @@ def export_spans(spans: list[Span], timeout_seconds: float = 3.0) -> bool: try: deadline = time.monotonic() + max(timeout_seconds, 0.1) token = _oidc_token(deadline) - for offset in range(0, len(spans), MAX_BATCH_SIZE): + batches: list[list[Span]] = [] + batch: list[Span] = [] + size = 0 + for span in spans: + span_size = len(_encode_span(span)) + if batch and ( + size + span_size > MAX_BATCH_BYTES or len(batch) >= MAX_BATCH_SIZE + ): + batches.append(batch) + batch, size = [], 0 + batch.append(span) + size += span_size + if batch: + batches.append(batch) + for batch in batches: request = urllib.request.Request( ENDPOINT, - data=encode_request(spans[offset : offset + MAX_BATCH_SIZE]), + data=encode_request(batch), method="POST", headers={ "Authorization": f"Bearer {token}", diff --git a/.buildkite/scripts/ci-otel/ci_otel.sh b/.buildkite/scripts/ci-otel/ci_otel.sh index feec16fa65..78b18207ad 100755 --- a/.buildkite/scripts/ci-otel/ci_otel.sh +++ b/.buildkite/scripts/ci-otel/ci_otel.sh @@ -82,6 +82,13 @@ ci_otel_start() { CI_INFRA_COMMAND_SPAN_ID="${_CI_INFRA_OTEL_COMMAND_SPAN_ID}" export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID _CI_INFRA_OTEL_ACTIVE=1 + _CI_INFRA_GPU_PID="" + if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] && + [ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] && + command -v nvidia-smi >/dev/null 2>&1; then + "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" /dev/null || : + _CI_INFRA_GPU_WAIT=0 + while kill -0 "${_CI_INFRA_GPU_PID}" 2>/dev/null && + [ "${_CI_INFRA_GPU_WAIT}" -lt 30 ]; do + sleep 0.1 + _CI_INFRA_GPU_WAIT=$((_CI_INFRA_GPU_WAIT + 1)) + done + kill -KILL "${_CI_INFRA_GPU_PID}" 2>/dev/null || : + wait "${_CI_INFRA_GPU_PID}" 2>/dev/null || : + _CI_INFRA_GPU_PID="" + fi CI_INFRA_TRACE_ID="" CI_INFRA_COMMAND_SPAN_ID="" export CI_INFRA_TRACE_ID CI_INFRA_COMMAND_SPAN_ID diff --git a/.buildkite/scripts/ci-otel/tests/test_ci_otel.py b/.buildkite/scripts/ci-otel/tests/test_ci_otel.py index ff279de953..f636e7ed18 100644 --- a/.buildkite/scripts/ci-otel/tests/test_ci_otel.py +++ b/.buildkite/scripts/ci-otel/tests/test_ci_otel.py @@ -8,6 +8,7 @@ import os import shlex import subprocess import sys +import threading import time from pathlib import Path @@ -16,6 +17,7 @@ import pytest SCRIPTS_DIR = Path(__file__).resolve().parents[1] sys.path.insert(0, str(SCRIPTS_DIR)) +import ci_gpu # noqa: E402 import ci_otel # noqa: E402 from ci_otel import Span # noqa: E402 @@ -251,9 +253,15 @@ def test_export_failure_is_soft_and_bounded(monkeypatch): assert time.monotonic() - started < 0.5 -def test_export_batches_with_one_oidc_token(monkeypatch): +@pytest.mark.parametrize("byte_limited", [False, True]) +def test_export_batches_with_one_oidc_token(monkeypatch, byte_limited): monkeypatch.setenv("BUILDKITE", "true") - monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2) + if byte_limited: + monkeypatch.setattr( + ci_otel, "MAX_BATCH_BYTES", len(ci_otel._encode_span(_span())) * 2 + ) + else: + monkeypatch.setattr(ci_otel, "MAX_BATCH_SIZE", 2) token_calls = [] requests = [] @@ -786,3 +794,127 @@ def test_pytest_plugin_failure_cannot_fail_pytest(tmp_path): assert result.returncode == 0, result.stderr assert "1 passed" in result.stdout + + +def test_gpu_samples_preserve_zero_and_omit_unsupported_metrics(): + events = ci_gpu.parse_samples( + "0, GPU-a, H200, 0, 1024, 2048, Disabled\n" + "1, GPU-b, H200, [N/A], [N/A], 2048, Disabled\n" + "2, GPU-c, H200, 50, 1024, 2048, Enabled\n" + "3, GPU-d, H200, NaN, -1, inf, Disabled\n", + 1234, + ) + assert len(events) == 4 + assert events[0]["attributes"]["gpu.utilization"] == 0 + assert events[0]["attributes"]["gpu.memory.used"] == 1024**3 + for event in events[1:]: + assert "gpu.utilization" not in event["attributes"] + assert "gpu.memory.used" not in event["attributes"] + assert events[2]["attributes"]["gpu.status"] == "unsupported_mig" + assert "gpu.memory.total" not in events[2]["attributes"] + + +def test_gpu_batches_keep_command_identity_and_survive_spool_roundtrip( + monkeypatch, tmp_path +): + monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16) + monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8) + monkeypatch.setenv("CI_INFRA_OTEL_SPOOL_DIR", str(tmp_path)) + span = ci_gpu.sample_span( + ci_gpu.parse_samples("0, GPU-a, H200, 90, 1, 2, Disabled", 1234) + ) + assert span.parent_span_id == "02" * 8 + assert span.trace_id == "01" * 16 + assert ci_otel.record_spans([span]) + assert ci_otel.load_spans() == [span] + assert b"ci.gpu.sample" in ci_otel.encode_request([span]) + + +def test_gpu_query_failures_are_bounded_and_never_synthesize_idle_samples(monkeypatch): + calls = [] + + def query(*args, **kwargs): + calls.append(kwargs["timeout"]) + raise subprocess.TimeoutExpired("nvidia-smi", kwargs["timeout"]) + + monkeypatch.setattr(ci_gpu.subprocess, "run", query) + monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0) + monkeypatch.setattr( + ci_gpu, "record_spans", lambda _: pytest.fail("fabricated sample") + ) + ci_gpu.collect(os.getppid(), threading.Event()) + assert calls == [2, 2, 2] + + +def test_gpu_periodic_batches_are_bounded_even_when_upload_fails(monkeypatch): + monkeypatch.setenv("CI_INFRA_TRACE_ID", "01" * 16) + monkeypatch.setenv("CI_INFRA_COMMAND_SPAN_ID", "02" * 8) + monkeypatch.setattr(ci_gpu, "BATCH_SECONDS", 0) + monkeypatch.setattr(ci_gpu, "INTERVAL_SECONDS", 0) + monkeypatch.setattr( + ci_gpu.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess( + a, 0, "0, GPU-a, H200, 50, 1, 2, Disabled" + ), + ) + stop = threading.Event() + batches = [] + + def upload(spans, timeout_seconds): + batches.append(spans) + assert timeout_seconds == 2 + if len(batches) == 3: + stop.set() + return False + + monkeypatch.setattr(ci_gpu, "export_spans", upload) + ci_gpu.collect(os.getppid(), stop) + assert [len(batch[0].events) for batch in batches] == [1, 1, 1] + + +@pytest.mark.parametrize("sampling", ["1", "0"]) +def test_gpu_sampler_stops_with_command_and_preserves_failure_status( + tmp_path, sampling +): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + smi = bin_dir / "nvidia-smi" + smi.write_text('#!/bin/sh\necho "0, GPU-a, H200, 70, 1024, 2048, Disabled"\n') + smi.chmod(0o755) + # Keep the spool for inspection, and avoid any network upload. + shell = ( + f'. "{SCRIPTS_DIR / "ci_otel.sh"}"\n' + "ci_otel_run 1 workload sh -c 'sleep 1.3; exit 7'\n" + "exit $?" + ) + result = subprocess.run( + ["/bin/sh", "-c", shell], + capture_output=True, + text=True, + timeout=8, + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "BUILDKITE": "false", + "CI_INFRA_GPU_SAMPLING": sampling, + "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR), + "CI_INFRA_OTEL_RUNTIME_DIR": str(tmp_path / "runtime"), + "CI_INFRA_OTEL_SPOOL_DIR": str(tmp_path / "spans"), + }, + ) + assert result.returncode == 7, result.stderr + records = [ + json.loads(line) + for path in (tmp_path / "spans").glob("spans-*.jsonl") + for line in path.read_text().splitlines() + ] + command = next(record for record in records if record["name"] == "ci.command") + batches = [record for record in records if record["name"] == "ci.gpu.samples"] + assert bool(batches) == (sampling == "1") + if batches: + assert batches[0]["parent_span_id"] == command["span_id"] + assert all( + command["start_ns"] <= event["time_ns"] <= command["end_ns"] + for event in batches[0]["events"] + )