[perf] Use pinned memory for async H2D transfer in do_mamba_copy_block (#35480)

Signed-off-by: Huamin Li <3ericli@gmail.com>
This commit is contained in:
Huamin Li
2026-02-28 01:50:37 +08:00
committed by GitHub
parent 1d897ff04f
commit 157722da75
4 changed files with 85 additions and 44 deletions
+14
View File
@@ -755,6 +755,7 @@ class GPUModelRunner(
self.execute_model_state: ExecuteModelState | None = None
self.kv_connector_output: KVConnectorOutput | None = None
self.mamba_state_idx: dict[str, int] = {}
self._mamba_copy_bufs: mamba_utils.MambaCopyBuffers | None = None
self.layerwise_nvtx_hooks_registered = False
def update_max_model_len(self, max_model_len: int) -> None:
@@ -849,6 +850,16 @@ class GPUModelRunner(
with_numpy=numpy,
)
def _get_mamba_copy_bufs(self) -> mamba_utils.MambaCopyBuffers:
if self._mamba_copy_bufs is None:
self._mamba_copy_bufs = mamba_utils.MambaCopyBuffers.create(
self.max_num_reqs,
self.kv_cache_config,
self.model.get_mamba_state_copy_func(),
self._make_buffer,
)
return self._mamba_copy_bufs
def _init_model_kwargs(self):
model_kwargs = dict[str, Any]()
@@ -1211,6 +1222,7 @@ class GPUModelRunner(
self.mamba_state_idx,
self.compilation_config.static_forward_context,
self.model.get_mamba_state_copy_func(),
self._get_mamba_copy_bufs(),
)
def _update_streaming_request(
@@ -3505,6 +3517,7 @@ class GPUModelRunner(
self.requests,
self.compilation_config.static_forward_context,
self.model.get_mamba_state_copy_func(),
self._get_mamba_copy_bufs(),
)
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
@@ -5997,6 +6010,7 @@ class GPUModelRunner(
"""
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
self._mamba_copy_bufs = None
self.may_add_encoder_only_layers_to_kv_cache_config()
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
self.initialize_attn_backend(kv_cache_config)