[ROCm] Fix CI runtime and tests for MI355 DPX (#58244)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Mahesh Kunreddi <mahesh.kunreddi@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Mahesh Kunreddi
Claude
Andreas Karatzas
Codex
Andreas Karatzas
parent
44257c2e35
commit
4013f3ad6c
@@ -605,6 +605,33 @@ initialize_native_environment() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_dpx_gpu_exclusivity() {
|
||||
local devices=(/dev/dri/renderD*)
|
||||
local device_id lock_status
|
||||
|
||||
if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then
|
||||
echo "DPX guard requires exactly one render device; refusing to start tests." >&2
|
||||
return 1
|
||||
fi
|
||||
# This queue mounts the same host-local HF cache into every pod on the node.
|
||||
if ! mountpoint -q "${HF_HOME}"; then
|
||||
echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2
|
||||
return 1
|
||||
fi
|
||||
device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1
|
||||
mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1
|
||||
# Hold the descriptor through workload and teardown; never delete the file.
|
||||
exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1
|
||||
flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0
|
||||
lock_status=$?
|
||||
if [[ "${lock_status}" -eq 75 ]]; then
|
||||
echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2
|
||||
else
|
||||
echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
run_native_preflight() {
|
||||
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
|
||||
@@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() {
|
||||
# Bus data identifies a card within the public node without publishing its
|
||||
# persistent UUID, serial number, or process list.
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi static -b -g all
|
||||
amd-smi static --bus --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -e -k -P -x -g all
|
||||
amd-smi metric --ecc --ecc-blocks --pcie --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi bad-pages -p -r -u -g all
|
||||
amd-smi static --ras --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -p -t -u -m -v -g all
|
||||
amd-smi metric --power --temperature --usage --mem-usage --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi xgmi -l -g all
|
||||
amd-smi xgmi --link-status --gpu all
|
||||
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
|
||||
&& command -v rocm-smi >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
|
||||
@@ -1485,6 +1512,10 @@ if is_native_runtime; then
|
||||
echo "Failed to initialize the native test environment"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \
|
||||
&& "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then
|
||||
check_dpx_gpu_exclusivity || exit 1
|
||||
fi
|
||||
if [[ "${commands}" == *python_only_compile.sh* ]]; then
|
||||
# This no-GPU job validates the ROCm precompiled/editable install path,
|
||||
# rather than CPU runtime platform selection.
|
||||
|
||||
@@ -364,7 +364,7 @@ def test_end_to_end(model: str):
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep():
|
||||
def test_deep_sleep(gpu_memory_cleared):
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
@@ -395,7 +395,7 @@ def test_deep_sleep():
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_lora():
|
||||
def test_deep_sleep_lora(gpu_memory_cleared):
|
||||
"""Level-2 sleep/wake/reload with enable_lora=True.
|
||||
|
||||
LoRA wrapping moves parameters under base_layer and adds LoRA
|
||||
@@ -488,7 +488,7 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async():
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
@@ -527,7 +527,9 @@ def test_deep_sleep_async():
|
||||
|
||||
|
||||
@requires_fp8
|
||||
def test_deep_sleep_fp8_kvcache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_deep_sleep_fp8_kvcache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
|
||||
):
|
||||
# Regression test for https://github.com/vllm-project/vllm/pull/28783.
|
||||
# In particular, verify that MRV1 does not rely on post_kv_cache_wake_up()
|
||||
# to restore correct output after level-2 sleep.
|
||||
|
||||
@@ -1790,6 +1790,24 @@ def disable_deepgemm_ue8m0(monkeypatch):
|
||||
is_deep_gemm_e8m0_used.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gpu_memory_cleared():
|
||||
"""Wait for prior tests to release GPU memory on gfx950."""
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if on_gfx950():
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=[0],
|
||||
threshold_ratio=0.08,
|
||||
timeout_s=30,
|
||||
stable_duration_s=1,
|
||||
)
|
||||
|
||||
|
||||
def _should_clean_gpu_memory_between_tests() -> bool:
|
||||
# This must stay opt-in: a function-scoped fixture cannot distinguish
|
||||
# stale VRAM from allocations owned by longer-lived module/session fixtures.
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
Qwen3-0.6B-FP8.yaml
|
||||
Qwen3-1.7B-MXFP4.yaml
|
||||
@@ -12,6 +12,9 @@ from tests.kernels.quant_utils import (
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.get_device_capability() < (7, 0):
|
||||
@@ -64,7 +67,7 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
topk_ids = topk_ids.view(-1)
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
a_q, a_s = native_per_token_group_quant_int8(a, block_k)
|
||||
a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False)
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
@@ -72,7 +75,9 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_int8(
|
||||
act_out, block_k, round_to_nearest=False
|
||||
)
|
||||
act_out = act_out.to(torch.float32)
|
||||
out[mask] = native_w8a8_block_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
|
||||
@@ -132,3 +137,27 @@ def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
|
||||
# Check results
|
||||
torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_block_int8_quantization_truncates_fractional_values(dtype):
|
||||
"""Block INT8 quantization truncates, unlike per-token round-to-nearest."""
|
||||
device = current_platform.device_type
|
||||
x = torch.tensor(
|
||||
[[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122],
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
expected = torch.tensor(
|
||||
[[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device
|
||||
)
|
||||
|
||||
out_q, scale = per_token_group_quant_int8(x, group_size=128)
|
||||
ref_q, ref_s = native_per_token_group_quant_int8(
|
||||
x, group_size=128, round_to_nearest=False
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_s, scale, atol=0, rtol=0)
|
||||
|
||||
@@ -181,12 +181,15 @@ def native_per_token_group_quant_fp8(
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8):
|
||||
def native_per_token_group_quant_int8(
|
||||
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch.
|
||||
|
||||
It converts the tensor values into int8 values and returns the
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
Per-token quantization rounds to nearest; block quantization truncates.
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
@@ -201,9 +204,10 @@ def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8
|
||||
# Use float32 for scale calculation for stability
|
||||
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
|
||||
x_s = amax / int8_max
|
||||
x_q = (
|
||||
(x_.to(torch.float32) / x_s).round().clamp(min=int8_min, max=int8_max).to(dtype)
|
||||
) # Round before clamping
|
||||
x_q = x_.to(torch.float32) / x_s
|
||||
if round_to_nearest:
|
||||
x_q = x_q.round()
|
||||
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
|
||||
x_q = x_q.reshape(x.shape)
|
||||
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
|
||||
|
||||
|
||||
@@ -614,11 +614,13 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
# Accumulate in FP32, then match the GEMM's BF16 output rounding.
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float())
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float()
|
||||
|
||||
_print_close_stats(
|
||||
f"skinny_gemm M={M} N={N} K={K}",
|
||||
|
||||
@@ -132,7 +132,8 @@ M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
|
||||
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
|
||||
(128, 6144, 128),
|
||||
(96, 8192, 128),
|
||||
(128, 12288, 128),
|
||||
# Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs.
|
||||
(128, 12288, 64),
|
||||
(32, 12288, 128),
|
||||
]
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ def _set_conv_state_layout(monkeypatch, layout: str) -> None:
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
gpu_memory_cleared,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
|
||||
@@ -12,6 +12,7 @@ import gc
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -19,6 +20,8 @@ pytestmark = pytest.mark.skipif(
|
||||
reason="ROCm platform required",
|
||||
)
|
||||
|
||||
MAX_LEAK_PCT = 10
|
||||
|
||||
|
||||
def _mb(b: int) -> float:
|
||||
return b / (1024 * 1024)
|
||||
@@ -61,6 +64,18 @@ def _full_gpu_cleanup():
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
def _wait_for_driver_memory_release(baseline: dict, peak: dict) -> None:
|
||||
# Process exit can precede ROCm's release of the backing allocation.
|
||||
peak_delta_mb = max(0.0, peak["drv_used_mb"] - baseline["drv_used_mb"])
|
||||
target_mb = baseline["drv_used_mb"] + peak_delta_mb * MAX_LEAK_PCT / 100
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=[torch.accelerator.current_device_index()],
|
||||
threshold_bytes=int(target_mb * 1024**2),
|
||||
timeout_s=120,
|
||||
poll_interval_s=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)])
|
||||
def test_gpu_memory_nixl_hma(model_name, sw_size):
|
||||
"""Track GPU memory through NixlConnector create/infer/shutdown cycle."""
|
||||
@@ -116,7 +131,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
|
||||
|
||||
# shutdown + cleanup
|
||||
print("\n--- shutdown ---")
|
||||
llm.llm_engine.engine_core.shutdown()
|
||||
llm.llm_engine.engine_core.shutdown(timeout=60)
|
||||
_gpu_snapshot("3. after shutdown()", snap2["alloc_mb"])
|
||||
|
||||
del llm
|
||||
@@ -126,6 +141,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
|
||||
torch._dynamo.reset()
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
_wait_for_driver_memory_release(snap0, snap2)
|
||||
snap_final = _gpu_snapshot("4. final", snap2["alloc_mb"])
|
||||
|
||||
# summary
|
||||
@@ -160,7 +176,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
|
||||
# Peak driver memory used above baseline
|
||||
drv_peak = snap2["drv_used_mb"] - drv_base
|
||||
leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0
|
||||
max_leak_pct = 10
|
||||
max_leak_pct = MAX_LEAK_PCT
|
||||
assert leak_pct <= max_leak_pct, (
|
||||
f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory "
|
||||
f"not freed after NixlConnector shutdown "
|
||||
@@ -196,7 +212,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
|
||||
llm.generate(["hi " * 500], SamplingParams(max_tokens=1))
|
||||
snap_peak = _gpu_snapshot("after generate()", snap0["alloc_mb"])
|
||||
|
||||
llm.llm_engine.engine_core.shutdown()
|
||||
llm.llm_engine.engine_core.shutdown(timeout=60)
|
||||
del llm
|
||||
_full_gpu_cleanup()
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -204,6 +220,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
|
||||
torch._dynamo.reset()
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
_wait_for_driver_memory_release(snap0, snap_peak)
|
||||
snap_final = _gpu_snapshot("final", snap0["alloc_mb"])
|
||||
|
||||
drv_base = snap0["drv_used_mb"]
|
||||
@@ -213,7 +230,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
|
||||
print("=" * 90)
|
||||
|
||||
leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0
|
||||
max_leak_pct = 10
|
||||
max_leak_pct = MAX_LEAK_PCT
|
||||
assert leak_pct <= max_leak_pct, (
|
||||
f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory "
|
||||
f"not freed after baseline shutdown "
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501
|
||||
DummyStatLogger,
|
||||
)
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM
|
||||
from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger
|
||||
|
||||
@@ -30,8 +31,15 @@ async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args):
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger]
|
||||
)
|
||||
assert isinstance(engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger)
|
||||
engine.shutdown()
|
||||
try:
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger
|
||||
)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -48,14 +56,18 @@ async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args):
|
||||
disabled_log_engine_args, stat_loggers=[DummyStatLogger]
|
||||
)
|
||||
|
||||
assert len(engine.logger_manager.stat_loggers) == 2
|
||||
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
|
||||
DummyStatLogger,
|
||||
)
|
||||
try:
|
||||
assert len(engine.logger_manager.stat_loggers) == 2
|
||||
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
|
||||
assert isinstance(
|
||||
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
|
||||
DummyStatLogger,
|
||||
)
|
||||
|
||||
# log_stats is still True, since custom stat loggers are used
|
||||
assert engine.log_stats
|
||||
|
||||
engine.shutdown()
|
||||
# log_stats is still True, since custom stat loggers are used
|
||||
assert engine.log_stats
|
||||
finally:
|
||||
engine.shutdown()
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
import ray
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.config.model import ModelDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -46,31 +47,39 @@ def test_engine_log_metrics_ray(
|
||||
) -> None:
|
||||
"""Simple smoke test, verifying this can be used without exceptions.
|
||||
Need to start a Ray cluster in order to verify outputs."""
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True
|
||||
)
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class EngineTestActor:
|
||||
async def run(self):
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True
|
||||
)
|
||||
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
engine_args, stat_loggers=[RayPrometheusStatLogger]
|
||||
)
|
||||
|
||||
for i, prompt in enumerate(example_prompts):
|
||||
results = engine.generate(
|
||||
request_id=f"request-id-{i}",
|
||||
prompt=prompt,
|
||||
sampling_params=SamplingParams(max_tokens=max_tokens),
|
||||
)
|
||||
try:
|
||||
for i, prompt in enumerate(example_prompts):
|
||||
results = engine.generate(
|
||||
request_id=f"request-id-{i}",
|
||||
prompt=prompt,
|
||||
sampling_params=SamplingParams(max_tokens=max_tokens),
|
||||
)
|
||||
|
||||
async for _ in results:
|
||||
pass
|
||||
async for _ in results:
|
||||
pass
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
# Create the actor and call the async method
|
||||
actor = EngineTestActor.remote() # type: ignore[attr-defined]
|
||||
ray.get(actor.run.remote())
|
||||
try:
|
||||
actor = EngineTestActor.remote() # type: ignore[attr-defined]
|
||||
ray.get(actor.run.remote())
|
||||
finally:
|
||||
ray.shutdown()
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0 - engine_args.gpu_memory_utilization
|
||||
)
|
||||
|
||||
|
||||
def test_sanitized_opentelemetry_name():
|
||||
|
||||
Reference in New Issue
Block a user