[ROCm][CI] Fix cuda graph mem profile issue (#48764)
Signed-off-by: charlifu <charlifu@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
ba47bb5be1
commit
6a9f24aa8c
@@ -120,6 +120,7 @@ from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
current_stream,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
@@ -6608,17 +6609,19 @@ class GPUModelRunner(
|
||||
per_graph_estimate = {}
|
||||
encoder_memory_estimate = 0
|
||||
|
||||
# On ROCm, capture these throwaway profiling graphs on the current stream
|
||||
# instead of the fresh side stream graph_capture() allocates by default.
|
||||
# torch's allocator pools free blocks per stream, so a side-stream forward
|
||||
# strands a persistent aiter scratch buffer in a separate pool, shifting
|
||||
# the physical placement of the real KV cache allocated afterward and
|
||||
# slowing bandwidth-bound decode ~20%. The graphs are discarded, so a
|
||||
# side stream is unnecessary here.
|
||||
# cap_ctx=None keeps the side-stream path on CUDA, where the current
|
||||
# stream is the legacy default stream, on which capture cannot begin.
|
||||
# On ROCm, capture these throwaway profiling graphs on vLLM's dedicated
|
||||
# compute stream instead of the fresh side stream graph_capture()
|
||||
# allocates by default. torch's allocator pools free blocks per stream,
|
||||
# so a side-stream forward strands a persistent aiter scratch buffer in
|
||||
# a separate pool, shifting the physical placement of the real KV cache
|
||||
# allocated afterward and slowing bandwidth-bound decode ~20%. The
|
||||
# graphs are discarded, so a side stream is unnecessary here.
|
||||
# Use current_stream(), not torch.cuda.current_stream(): before vLLM
|
||||
# initializes its dedicated stream, torch returns the per-thread default
|
||||
# stream (cuda_stream=0), which cannot be used for cudagraph capture.
|
||||
# cap_ctx=None keeps the side-stream path on CUDA.
|
||||
cap_ctx = (
|
||||
GraphCaptureContext(torch.cuda.current_stream(self.device))
|
||||
GraphCaptureContext(current_stream())
|
||||
if current_platform.is_rocm()
|
||||
else None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user