[perf] Use pinned memory for async H2D transfer in do_mamba_copy_block (#35480)
Signed-off-by: Huamin Li <3ericli@gmail.com>
This commit is contained in:
@@ -755,6 +755,7 @@ class GPUModelRunner(
|
||||
self.execute_model_state: ExecuteModelState | None = None
|
||||
self.kv_connector_output: KVConnectorOutput | None = None
|
||||
self.mamba_state_idx: dict[str, int] = {}
|
||||
self._mamba_copy_bufs: mamba_utils.MambaCopyBuffers | None = None
|
||||
self.layerwise_nvtx_hooks_registered = False
|
||||
|
||||
def update_max_model_len(self, max_model_len: int) -> None:
|
||||
@@ -849,6 +850,16 @@ class GPUModelRunner(
|
||||
with_numpy=numpy,
|
||||
)
|
||||
|
||||
def _get_mamba_copy_bufs(self) -> mamba_utils.MambaCopyBuffers:
|
||||
if self._mamba_copy_bufs is None:
|
||||
self._mamba_copy_bufs = mamba_utils.MambaCopyBuffers.create(
|
||||
self.max_num_reqs,
|
||||
self.kv_cache_config,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._make_buffer,
|
||||
)
|
||||
return self._mamba_copy_bufs
|
||||
|
||||
def _init_model_kwargs(self):
|
||||
model_kwargs = dict[str, Any]()
|
||||
|
||||
@@ -1211,6 +1222,7 @@ class GPUModelRunner(
|
||||
self.mamba_state_idx,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_copy_bufs(),
|
||||
)
|
||||
|
||||
def _update_streaming_request(
|
||||
@@ -3505,6 +3517,7 @@ class GPUModelRunner(
|
||||
self.requests,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_copy_bufs(),
|
||||
)
|
||||
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
@@ -5997,6 +6010,7 @@ class GPUModelRunner(
|
||||
"""
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._mamba_copy_bufs = None
|
||||
self.may_add_encoder_only_layers_to_kv_cache_config()
|
||||
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
|
||||
self.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
Reference in New Issue
Block a user