[CI] Move compatible 1xL4 jobs to H200 35GB MIG (#43024)
Signed-off-by: Simon Mo <simon@inferact.ai> Co-authored-by: Simon Mo <simon@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Simon Mo
OpenAI Codex
OpenAI Codex
parent
ace9fda495
commit
1dcbbd9cac
@@ -18,6 +18,7 @@ steps:
|
|||||||
- pytest -v -s cuda/test_platform_no_cuda_init.py
|
- pytest -v -s cuda/test_platform_no_cuda_init.py
|
||||||
|
|
||||||
- label: Cudagraph
|
- label: Cudagraph
|
||||||
|
device: h200_35gb
|
||||||
key: cudagraph
|
key: cudagraph
|
||||||
timeout_in_minutes: 30
|
timeout_in_minutes: 30
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server OpenAI - Part 1)
|
- label: Entrypoints Integration (API Server OpenAI - Part 1)
|
||||||
|
device: h200_35gb
|
||||||
key: entrypoints-integration-api-server-openai-part-1
|
key: entrypoints-integration-api-server-openai-part-1
|
||||||
timeout_in_minutes: 45
|
timeout_in_minutes: 45
|
||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
@@ -75,6 +76,7 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server OpenAI - Part 2)
|
- label: Entrypoints Integration (API Server OpenAI - Part 2)
|
||||||
|
device: h200_35gb
|
||||||
key: entrypoints-integration-api-server-openai-part-2
|
key: entrypoints-integration-api-server-openai-part-2
|
||||||
timeout_in_minutes: 45
|
timeout_in_minutes: 45
|
||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
@@ -94,6 +96,7 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server Generate)
|
- label: Entrypoints Integration (API Server Generate)
|
||||||
|
device: h200_35gb
|
||||||
key: entrypoints-integration-api-server-generate
|
key: entrypoints-integration-api-server-generate
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
@@ -151,6 +154,7 @@ steps:
|
|||||||
- pytest -v -s entrypoints/multimodal
|
- pytest -v -s entrypoints/multimodal
|
||||||
|
|
||||||
- label: Entrypoints Integration (Pooling)
|
- label: Entrypoints Integration (Pooling)
|
||||||
|
device: h200_35gb
|
||||||
key: entrypoints-integration-pooling
|
key: entrypoints-integration-pooling
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ steps:
|
|||||||
- pytest -v -s tests/kernels/ir
|
- pytest -v -s tests/kernels/ir
|
||||||
|
|
||||||
- label: Kernels Core Operation Test
|
- label: Kernels Core Operation Test
|
||||||
|
device: h200_35gb
|
||||||
key: kernels-core-operation-test
|
key: kernels-core-operation-test
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -163,6 +164,7 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: Kernels Mamba Test
|
- label: Kernels Mamba Test
|
||||||
|
device: h200_35gb
|
||||||
key: kernels-mamba-test
|
key: kernels-mamba-test
|
||||||
timeout_in_minutes: 40
|
timeout_in_minutes: 40
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -64,8 +64,9 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: V1 Core + KV + Metrics
|
- label: V1 Core + KV + Metrics
|
||||||
|
device: h200_35gb
|
||||||
key: v1-core-kv-metrics
|
key: v1-core-kv-metrics
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 80
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/config/
|
- vllm/config/
|
||||||
- vllm/distributed/
|
- vllm/distributed/
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ depends_on:
|
|||||||
- image-build
|
- image-build
|
||||||
steps:
|
steps:
|
||||||
- label: Model Executor
|
- label: Model Executor
|
||||||
|
device: h200_35gb
|
||||||
key: model-executor
|
key: model-executor
|
||||||
timeout_in_minutes: 45
|
timeout_in_minutes: 45
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: Language Models Tests (Extra Standard) %N
|
- label: Language Models Tests (Extra Standard) %N
|
||||||
|
device: h200_35gb
|
||||||
key: language-models-tests-extra-standard
|
key: language-models-tests-extra-standard
|
||||||
timeout_in_minutes: 40
|
timeout_in_minutes: 40
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -51,8 +52,8 @@ steps:
|
|||||||
- tests/models/language/pooling/test_classification.py
|
- tests/models/language/pooling/test_classification.py
|
||||||
- vllm/_aiter_ops.py
|
- vllm/_aiter_ops.py
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
|
|
||||||
- label: Language Models Tests (Hybrid) %N
|
- label: Language Models Tests (Hybrid) %N
|
||||||
|
device: h200_35gb
|
||||||
key: language-models-tests-hybrid
|
key: language-models-tests-hybrid
|
||||||
timeout_in_minutes: 65
|
timeout_in_minutes: 65
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -63,8 +64,8 @@ steps:
|
|||||||
# Note: also needed to run plamo2 model in vLLM
|
# Note: also needed to run plamo2 model in vLLM
|
||||||
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
||||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||||
# Shard hybrid language model tests
|
# Shard the hybrid language model tests that are numerically stable on Hopper.
|
||||||
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
- pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||||
parallelism: 2
|
parallelism: 2
|
||||||
mirror:
|
mirror:
|
||||||
amd:
|
amd:
|
||||||
@@ -77,6 +78,20 @@ steps:
|
|||||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||||
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||||
|
|
||||||
|
# Granite 4 hybrid generation is sensitive to hardware-specific Triton SSD
|
||||||
|
# autotuning (https://github.com/vllm-project/vllm/issues/25194). Keep this one
|
||||||
|
# correctness test on L4 until its H200 output matches the Transformers reference.
|
||||||
|
- label: Language Models Tests (Granite L4 Compatibility)
|
||||||
|
key: language-models-tests-granite-l4-compatibility
|
||||||
|
timeout_in_minutes: 65
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/models/language/generation
|
||||||
|
commands:
|
||||||
|
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
||||||
|
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||||
|
- pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'
|
||||||
|
|
||||||
- label: Language Models Test (Extended Generation) # 80min
|
- label: Language Models Test (Extended Generation) # 80min
|
||||||
device: h200_35gb
|
device: h200_35gb
|
||||||
key: language-models-test-extended-generation
|
key: language-models-test-extended-generation
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ steps:
|
|||||||
- vllm/model_executor/model_loader/
|
- vllm/model_executor/model_loader/
|
||||||
|
|
||||||
- label: Multi-Modal Models (Extended Generation 1)
|
- label: Multi-Modal Models (Extended Generation 1)
|
||||||
|
device: h200_35gb
|
||||||
key: multi-modal-models-extended-generation-1
|
key: multi-modal-models-extended-generation-1
|
||||||
optional: true
|
optional: true
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -116,8 +116,9 @@ steps:
|
|||||||
- image-build-amd
|
- image-build-amd
|
||||||
|
|
||||||
- label: PyTorch Fullgraph Smoke Test
|
- label: PyTorch Fullgraph Smoke Test
|
||||||
|
device: h200_35gb
|
||||||
key: pytorch-fullgraph-smoke-test
|
key: pytorch-fullgraph-smoke-test
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 90
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/__init__.py
|
- vllm/__init__.py
|
||||||
- vllm/_aiter_ops.py
|
- vllm/_aiter_ops.py
|
||||||
@@ -149,7 +150,42 @@ steps:
|
|||||||
# as it is a heavy test that is covered in other steps.
|
# as it is a heavy test that is covered in other steps.
|
||||||
# Use `find` to launch multiple instances of pytest so that
|
# Use `find` to launch multiple instances of pytest so that
|
||||||
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965
|
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965
|
||||||
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
|
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
|
||||||
|
|
||||||
|
# Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
|
||||||
|
# instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
|
||||||
|
# those SM90 cases skip while the architecture-compatible cases still run.
|
||||||
|
- label: PyTorch Fullgraph CUDAGraph (L4 Compatibility)
|
||||||
|
key: pytorch-fullgraph-cudagraph-l4-compatibility
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/__init__.py
|
||||||
|
- vllm/_aiter_ops.py
|
||||||
|
- vllm/_custom_ops.py
|
||||||
|
- vllm/compilation/
|
||||||
|
- vllm/config/
|
||||||
|
- vllm/distributed/
|
||||||
|
- vllm/engine/
|
||||||
|
- vllm/env_override.py
|
||||||
|
- vllm/envs.py
|
||||||
|
- vllm/forward_context.py
|
||||||
|
- vllm/inputs/
|
||||||
|
- vllm/ir/
|
||||||
|
- vllm/kernels/
|
||||||
|
- vllm/logger.py
|
||||||
|
- vllm/model_executor/
|
||||||
|
- vllm/multimodal/
|
||||||
|
- vllm/platforms/
|
||||||
|
- vllm/plugins/
|
||||||
|
- vllm/sampling_params.py
|
||||||
|
- vllm/sequence.py
|
||||||
|
- vllm/transformers_utils/
|
||||||
|
- vllm/triton_utils/
|
||||||
|
- vllm/utils/
|
||||||
|
- vllm/v1/
|
||||||
|
- tests/compile
|
||||||
|
commands:
|
||||||
|
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
|
||||||
|
|
||||||
- label: PyTorch Fullgraph
|
- label: PyTorch Fullgraph
|
||||||
key: pytorch-fullgraph
|
key: pytorch-fullgraph
|
||||||
|
|||||||
@@ -3,8 +3,11 @@ depends_on:
|
|||||||
- image-build
|
- image-build
|
||||||
steps:
|
steps:
|
||||||
- label: Quantization
|
- label: Quantization
|
||||||
|
device: h200_35gb
|
||||||
key: quantization
|
key: quantization
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 75
|
||||||
|
env:
|
||||||
|
VLLM_USE_V2_MODEL_RUNNER: "0"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- csrc/
|
- csrc/
|
||||||
- vllm/model_executor/layers/quantization
|
- vllm/model_executor/layers/quantization
|
||||||
@@ -19,9 +22,13 @@ steps:
|
|||||||
# TODO(jerryzh168): resolve the above comment
|
# TODO(jerryzh168): resolve the above comment
|
||||||
- uv pip install --system torchao==0.17.0 --index-url https://download.pytorch.org/whl/cu130
|
- uv pip install --system torchao==0.17.0 --index-url https://download.pytorch.org/whl/cu130
|
||||||
- uv pip install --system conch-triton-kernels
|
- uv pip install --system conch-triton-kernels
|
||||||
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
|
# The SM90-only checkpoint currently contains a removed weight_chan_scale
|
||||||
|
# parameter. It was not exercised by the previous L4 job.
|
||||||
|
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||||
|
parallelism: 8
|
||||||
|
|
||||||
- label: Quantized Fusions
|
- label: Quantized Fusions
|
||||||
|
device: h200_35gb
|
||||||
key: quantized-fusions
|
key: quantized-fusions
|
||||||
timeout_in_minutes: 20
|
timeout_in_minutes: 20
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -52,10 +59,14 @@ steps:
|
|||||||
- pytest -s -v tests/quantization/test_blackwell_moe.py
|
- pytest -s -v tests/quantization/test_blackwell_moe.py
|
||||||
|
|
||||||
- label: Quantized Models Test
|
- label: Quantized Models Test
|
||||||
|
device: h200_35gb
|
||||||
key: quantized-models-test
|
key: quantized-models-test
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 65
|
||||||
|
env:
|
||||||
|
VLLM_USE_V2_MODEL_RUNNER: "0"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/model_executor/layers/quantization
|
- vllm/model_executor/layers/quantization
|
||||||
- tests/models/quantization
|
- tests/models/quantization
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s models/quantization
|
- pytest -v -s models/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||||
|
parallelism: 3
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ steps:
|
|||||||
- pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
|
- pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
|
||||||
|
|
||||||
- label: Rust Frontend Tool Use
|
- label: Rust Frontend Tool Use
|
||||||
|
device: h200_35gb
|
||||||
timeout_in_minutes: 25
|
timeout_in_minutes: 25
|
||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -347,7 +347,11 @@ def test_selective_state_update(dim, dstate, has_z, itype):
|
|||||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||||
if itype == torch.bfloat16:
|
if itype == torch.bfloat16:
|
||||||
rtol, atol = 1e-2, 5e-2
|
rtol, atol = 1e-2, 5e-2
|
||||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
if (
|
||||||
|
current_platform.is_rocm()
|
||||||
|
or current_platform.is_xpu()
|
||||||
|
or current_platform.is_device_capability_family(90)
|
||||||
|
):
|
||||||
atol *= 2
|
atol *= 2
|
||||||
# set seed
|
# set seed
|
||||||
set_random_seed(0)
|
set_random_seed(0)
|
||||||
|
|||||||
@@ -188,6 +188,16 @@ def test_models(
|
|||||||
|
|
||||||
prompt_embeds.append(embed.squeeze(0))
|
prompt_embeds.append(embed.squeeze(0))
|
||||||
|
|
||||||
|
vllm_kwargs = {}
|
||||||
|
if (
|
||||||
|
model == "bigscience/bloom-560m"
|
||||||
|
and current_platform.is_device_capability_family(90)
|
||||||
|
):
|
||||||
|
# On SM90, the metadata builder otherwise selects FA3 AOT scheduling
|
||||||
|
# before Bloom's ALiBi layers fall back to FA2. Pinning FA2 keeps the
|
||||||
|
# builder and layer consistent and preserves the L4 test path.
|
||||||
|
vllm_kwargs["attention_config"] = {"flash_attn_version": 2}
|
||||||
|
|
||||||
with vllm_runner(
|
with vllm_runner(
|
||||||
model,
|
model,
|
||||||
tokenizer_name=model_info.tokenizer or model,
|
tokenizer_name=model_info.tokenizer or model,
|
||||||
@@ -200,6 +210,7 @@ def test_models(
|
|||||||
max_num_seqs=1 if current_platform.is_rocm() else 2,
|
max_num_seqs=1 if current_platform.is_rocm() else 2,
|
||||||
enable_prompt_embeds=use_prompt_embeds,
|
enable_prompt_embeds=use_prompt_embeds,
|
||||||
compilation_config={"cudagraph_capture_sizes": [1, 2]},
|
compilation_config={"cudagraph_capture_sizes": [1, 2]},
|
||||||
|
**vllm_kwargs,
|
||||||
) as vllm_model:
|
) as vllm_model:
|
||||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||||
example_prompts, max_tokens, num_logprobs
|
example_prompts, max_tokens, num_logprobs
|
||||||
|
|||||||
@@ -384,8 +384,13 @@ def test_fp32_cache_state(
|
|||||||
example_prompts, max_tokens, num_logprobs
|
example_prompts, max_tokens, num_logprobs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Leave enough headroom for repeated engine initialization on a
|
||||||
|
# 32.5 GiB MIG.
|
||||||
with vllm_runner(
|
with vllm_runner(
|
||||||
model, max_num_seqs=MAX_NUM_SEQS, **{cache_dtype_param: "float32"}
|
model,
|
||||||
|
max_num_seqs=MAX_NUM_SEQS,
|
||||||
|
gpu_memory_utilization=0.9,
|
||||||
|
**{cache_dtype_param: "float32"},
|
||||||
) as vllm_model:
|
) as vllm_model:
|
||||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||||
example_prompts, max_tokens, num_logprobs
|
example_prompts, max_tokens, num_logprobs
|
||||||
|
|||||||
@@ -67,6 +67,8 @@ def test_models(
|
|||||||
|
|
||||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
||||||
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
|
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
|
||||||
|
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_cuda():
|
||||||
|
pytest.skip(f"{kv_cache_dtype} is not supported by FLASH_ATTN on CUDA.")
|
||||||
|
|
||||||
if not (
|
if not (
|
||||||
current_platform.is_xpu()
|
current_platform.is_xpu()
|
||||||
|
|||||||
@@ -93,6 +93,9 @@ def test_online_quantization(
|
|||||||
use_rocm_aiter: bool,
|
use_rocm_aiter: bool,
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||||
|
pytest.skip("FA3 currently rejects FP8 KV cache output dtype on SM90")
|
||||||
|
|
||||||
if use_rocm_aiter:
|
if use_rocm_aiter:
|
||||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||||
|
|
||||||
|
|||||||
@@ -717,7 +717,10 @@ def test_triton_unified_attention_per_token_head_scale(
|
|||||||
|
|
||||||
# Coarser quantization → wider tolerance.
|
# Coarser quantization → wider tolerance.
|
||||||
if is_int4:
|
if is_int4:
|
||||||
atol, rtol = 0.5, 0.5
|
# Hopper's attention reduction order can move a few BF16 elements by
|
||||||
|
# just over 1.0 after INT4 quantization.
|
||||||
|
atol = 1.1 if current_platform.is_device_capability_family(90) else 0.5
|
||||||
|
rtol = 0.5
|
||||||
else:
|
else:
|
||||||
atol, rtol = 5e-2, 5e-2
|
atol, rtol = 5e-2, 5e-2
|
||||||
torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)
|
torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)
|
||||||
|
|||||||
+14
-1
@@ -108,6 +108,7 @@ if current_platform.is_rocm():
|
|||||||
elif current_platform.is_cuda():
|
elif current_platform.is_cuda():
|
||||||
from vllm.third_party.pynvml import (
|
from vllm.third_party.pynvml import (
|
||||||
nvmlDeviceGetHandleByIndex,
|
nvmlDeviceGetHandleByIndex,
|
||||||
|
nvmlDeviceGetHandleByUUID,
|
||||||
nvmlDeviceGetMemoryInfo,
|
nvmlDeviceGetMemoryInfo,
|
||||||
nvmlInit,
|
nvmlInit,
|
||||||
nvmlShutdown,
|
nvmlShutdown,
|
||||||
@@ -1521,6 +1522,18 @@ def get_physical_device_indices(devices: list[int]):
|
|||||||
return [index_mapping[i] for i in devices if i in index_mapping]
|
return [index_mapping[i] for i in devices if i in index_mapping]
|
||||||
|
|
||||||
|
|
||||||
|
def get_nvml_device_handle(device: int):
|
||||||
|
visible_devices = os.environ.get("NVIDIA_VISIBLE_DEVICES")
|
||||||
|
if visible_devices is not None:
|
||||||
|
identifiers = visible_devices.split(",")
|
||||||
|
if device < len(identifiers):
|
||||||
|
identifier = identifiers[device]
|
||||||
|
if identifier.startswith(("GPU-", "MIG-")):
|
||||||
|
return nvmlDeviceGetHandleByUUID(identifier)
|
||||||
|
|
||||||
|
return nvmlDeviceGetHandleByIndex(device)
|
||||||
|
|
||||||
|
|
||||||
@_nvml()
|
@_nvml()
|
||||||
def record_gpu_memory_usage_stats(
|
def record_gpu_memory_usage_stats(
|
||||||
*,
|
*,
|
||||||
@@ -1534,7 +1547,7 @@ def record_gpu_memory_usage_stats(
|
|||||||
gb_used = mem_info["vram_used"] / 2**10
|
gb_used = mem_info["vram_used"] / 2**10
|
||||||
gb_total = mem_info["vram_total"] / 2**10
|
gb_total = mem_info["vram_total"] / 2**10
|
||||||
else:
|
else:
|
||||||
dev_handle = nvmlDeviceGetHandleByIndex(device)
|
dev_handle = get_nvml_device_handle(device)
|
||||||
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
||||||
gb_used = mem_info.used / 2**30
|
gb_used = mem_info.used / 2**30
|
||||||
gb_total = mem_info.total / 2**30
|
gb_total = mem_info.total / 2**30
|
||||||
|
|||||||
Reference in New Issue
Block a user