[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2 (#27532)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -162,6 +162,7 @@ from vllm.v1.worker.ubatch_utils import (
|
||||
maybe_create_ubatch_slices,
|
||||
)
|
||||
from vllm.v1.worker.utils import is_residual_scattered_for_sp
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
from .utils import (
|
||||
AttentionGroup,
|
||||
@@ -297,6 +298,7 @@ class GPUModelRunner(
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory_available()
|
||||
self.dtype = self.model_config.dtype
|
||||
|
||||
self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(
|
||||
cache_config.cache_dtype, self.model_config
|
||||
)
|
||||
@@ -4597,6 +4599,10 @@ class GPUModelRunner(
|
||||
# after here.
|
||||
set_cudagraph_capturing_enabled(False)
|
||||
|
||||
# Lock workspace to prevent resizing during execution.
|
||||
# Max workspace sizes should have been captured during warmup/profiling.
|
||||
lock_workspace()
|
||||
|
||||
end_time = time.perf_counter()
|
||||
elapsed_time = end_time - start_time
|
||||
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
|
||||
|
||||
Reference in New Issue
Block a user