[CI] Support FORCE_COLOR env var; enable colors in CPU CI output (#56913)

Signed-off-by: Nick Hill <nickhill123@gmail.com>

.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh

@@ -30,7 +30,7 @@ remove_docker_container
 docker pull "$IMAGE"
 
 # Run the image
-docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE"
+docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE"
 
 print_packages() {
   docker exec "$CONTAINER_NAME" bash -c "

.buildkite/scripts/hardware_ci/run-cpu-test.sh

@@ -99,7 +99,7 @@ rm -f "$build_log"
 OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with'
 run_test() {
     local hf_offline=$1
-    docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
+    docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g "$IMAGE_NAME" \
         timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
 }
 

tests/test_logger.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import enum
+import io
 import json
 import logging
 import os
@@ -19,6 +20,7 @@ from vllm.logger import (
     _DATE_FORMAT,
     _FORMAT,
     _configure_vllm_root_logger,
+    _use_color,
     enable_trace_function_call,
     init_logger,
 )
@@ -72,6 +74,27 @@ def test_default_vllm_root_logger_configuration(monkeypatch):
     assert formatter.datefmt == _DATE_FORMAT
 
 
+def test_use_color_force_color(monkeypatch):
+    """FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an
+    explicit VLLM_LOGGING_COLOR=0 take precedence over it."""
+    monkeypatch.setattr(sys, "stdout", io.StringIO())
+    monkeypatch.setattr(sys, "stderr", io.StringIO())
+    for var in ("NO_COLOR", "FORCE_COLOR", "VLLM_LOGGING_COLOR"):
+        monkeypatch.delenv(var, raising=False)
+
+    assert not _use_color()
+
+    monkeypatch.setenv("FORCE_COLOR", "1")
+    assert _use_color()
+
+    monkeypatch.setenv("VLLM_LOGGING_COLOR", "0")
+    assert not _use_color()
+    monkeypatch.delenv("VLLM_LOGGING_COLOR")
+
+    monkeypatch.setenv("NO_COLOR", "1")
+    assert not _use_color()
+
+
 def test_descendent_loggers_depend_on_and_propagate_logs_to_root_logger(monkeypatch):
     """This test presumes that VLLM_CONFIGURE_LOGGING (default: True) and
     VLLM_LOGGING_CONFIG_PATH (default: None) are not configured and default

vllm/envs.py

@@ -44,6 +44,7 @@ if TYPE_CHECKING:
     VLLM_LOGGING_CONFIG_PATH: str | None = None
     VLLM_LOGGING_COLOR: str = "auto"
     NO_COLOR: bool = False
+    FORCE_COLOR: bool = False
     VLLM_LOG_STATS_INTERVAL: float = 10.0
     VLLM_TRACE_FUNCTION: int = 0
     VLLM_USE_FLASHINFER_SAMPLER: bool = True
@@ -840,6 +841,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
     "VLLM_LOGGING_COLOR": lambda: os.getenv("VLLM_LOGGING_COLOR", "auto"),
     # Standard unix flag for disabling ANSI color codes
     "NO_COLOR": lambda: os.getenv("NO_COLOR", "0") != "0",
+    # De-facto standard flag for forcing ANSI color codes (e.g. non-tty case)
+    "FORCE_COLOR": lambda: os.getenv("FORCE_COLOR", "0") != "0",
     # If set, vllm will log stats at this interval in seconds
     # If not set, vllm will log stats every 10 seconds.
     "VLLM_LOG_STATS_INTERVAL": lambda: (
@@ -2335,6 +2338,7 @@ def compile_factors() -> dict[str, object]:
         "CUDA_VISIBLE_DEVICES",
         "NO_COLOR",
         "VLLM_SKIP_VERSION_SUFFIX",
+        "FORCE_COLOR",
     }
 
     from vllm.config.utils import normalize_value

vllm/logger.py

@@ -29,7 +29,7 @@ _DATE_FORMAT = "%m-%d %H:%M:%S"
 def _use_color() -> bool:
     if envs.NO_COLOR or envs.VLLM_LOGGING_COLOR == "0":
         return False
-    if envs.VLLM_LOGGING_COLOR == "1":
+    if envs.VLLM_LOGGING_COLOR == "1" or envs.FORCE_COLOR:
         return True
     if envs.VLLM_LOGGING_STREAM == "ext://sys.stdout":  # stdout
         return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()