[Perf] [Hybrid] Fused Triton kernel for GPU-side Mamba state postprocessing (#40172)
Signed-off-by: Francesco Fusco <ffu@zurich.ibm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c68c55d43e
commit
b730c46352
@@ -887,7 +887,7 @@ class GPUModelRunner(
|
||||
self.execute_model_state: ExecuteModelState | None = None
|
||||
self.kv_connector_output: KVConnectorOutput | None = None
|
||||
self.mamba_state_idx: dict[str, int] = {}
|
||||
self._mamba_copy_bufs: mamba_utils.MambaCopyBuffers | None = None
|
||||
self._mamba_bufs: mamba_utils.MambaBuffers | None = None
|
||||
self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None
|
||||
if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0:
|
||||
self.mamba_prev_last_scheduled_idx = self._make_buffer(
|
||||
@@ -992,15 +992,23 @@ class GPUModelRunner(
|
||||
with_numpy=numpy,
|
||||
)
|
||||
|
||||
def _get_mamba_copy_bufs(self) -> mamba_utils.MambaCopyBuffers:
|
||||
if self._mamba_copy_bufs is None:
|
||||
self._mamba_copy_bufs = mamba_utils.MambaCopyBuffers.create(
|
||||
self.max_num_reqs,
|
||||
self.kv_cache_config,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._make_buffer,
|
||||
def _get_mamba_bufs(self) -> mamba_utils.MambaBuffers:
|
||||
# Only reachable on the ``mamba_cache_mode == "align"`` path.
|
||||
# The postprocess sub-object is additionally gated on spec
|
||||
# decode + hybrid model.
|
||||
assert self.cache_config.mamba_cache_mode == "align"
|
||||
if self._mamba_bufs is None:
|
||||
self._mamba_bufs = mamba_utils.MambaBuffers.create(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
copy_funcs=self.model.get_mamba_state_copy_func(),
|
||||
make_buffer=self._make_buffer,
|
||||
device=self.device,
|
||||
with_postprocess_align=(
|
||||
self.speculative_config is not None and self.model_config.is_hybrid
|
||||
),
|
||||
)
|
||||
return self._mamba_copy_bufs
|
||||
return self._mamba_bufs
|
||||
|
||||
def _init_model_kwargs(self):
|
||||
model_kwargs = dict[str, Any]()
|
||||
@@ -1489,21 +1497,32 @@ class GPUModelRunner(
|
||||
if not self.speculative_config or not self.model_config.is_hybrid:
|
||||
return
|
||||
|
||||
# TODO: Remove .cpu() sync to enable fully async for hybrid model;
|
||||
# Use num_computed_tokens.gpu instead of req.num_computed_tokens to
|
||||
# support aligned mamba cache mode.
|
||||
# Count the number of accepted tokens for each sequence.
|
||||
# Valid tokens are contiguous from position 0, so counting non-(-1)
|
||||
# tokens gives us the first -1 position (i.e., number of accepted).
|
||||
num_reqs = output_token_ids.size(0)
|
||||
self.num_accepted_tokens.gpu[:num_reqs] = (output_token_ids != -1).sum(dim=1)
|
||||
|
||||
is_align = self.cache_config.mamba_cache_mode == "align"
|
||||
if is_align:
|
||||
for i, num_tokens in enumerate(
|
||||
self.num_accepted_tokens.gpu[:num_reqs].cpu().numpy()
|
||||
):
|
||||
self.input_batch.num_accepted_tokens_cpu[i] = num_tokens
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
# Fused GPU postprocess: state copies + per-request accepted-token
|
||||
# update without CPU-GPU sync. The metadata
|
||||
# (num_scheduled_tokens, num_draft_tokens, num_computed_tokens) is
|
||||
# pre-staged to GPU buffers in _prepare_inputs.
|
||||
mamba_utils.postprocess_mamba_align_gpu(
|
||||
bufs=self._get_mamba_bufs(),
|
||||
num_reqs=num_reqs,
|
||||
num_accepted_tokens_gpu=self.num_accepted_tokens.gpu,
|
||||
num_accepted_tokens_cpu_tensor=(
|
||||
self.input_batch.num_accepted_tokens_cpu_tensor
|
||||
),
|
||||
input_batch=self.input_batch,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
forward_context=self.compilation_config.static_forward_context,
|
||||
mamba_state_copy_funcs=self.model.get_mamba_state_copy_func(),
|
||||
)
|
||||
|
||||
assert self.num_accepted_tokens_event is not None
|
||||
self.num_accepted_tokens_event.record()
|
||||
else:
|
||||
self.input_batch.num_accepted_tokens_cpu_tensor[:num_reqs].copy_(
|
||||
self.num_accepted_tokens.gpu[:num_reqs], non_blocking=True
|
||||
@@ -1511,23 +1530,16 @@ class GPUModelRunner(
|
||||
assert self.num_accepted_tokens_event is not None
|
||||
self.num_accepted_tokens_event.record()
|
||||
|
||||
mamba_utils.postprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.num_spec_tokens,
|
||||
num_reqs,
|
||||
forward_context=(
|
||||
self.compilation_config.static_forward_context if is_align else None
|
||||
),
|
||||
mamba_state_copy_funcs=(
|
||||
self.model.get_mamba_state_copy_func() if is_align else None
|
||||
),
|
||||
copy_bufs=self._get_mamba_copy_bufs() if is_align else None,
|
||||
)
|
||||
if self.cache_config.mamba_cache_mode == "all":
|
||||
mamba_utils.postprocess_mamba_all(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.num_spec_tokens,
|
||||
num_reqs,
|
||||
)
|
||||
|
||||
def _update_streaming_request(
|
||||
self, req_id: str, new_req_data: NewRequestData
|
||||
@@ -4098,6 +4110,7 @@ class GPUModelRunner(
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
deferred_state_corrections_fn = None
|
||||
mamba_bufs = self._get_mamba_bufs()
|
||||
mamba_utils.preprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
@@ -4107,7 +4120,7 @@ class GPUModelRunner(
|
||||
self.requests,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_copy_bufs(),
|
||||
mamba_bufs.preprocess,
|
||||
)
|
||||
# preprocess_mamba resets num_accepted_tokens_cpu to 1
|
||||
# for requests whose state was copied to a new block.
|
||||
@@ -4118,6 +4131,21 @@ class GPUModelRunner(
|
||||
)
|
||||
self.num_accepted_tokens.copy_to_gpu(num_reqs)
|
||||
|
||||
# Stage per-request inputs for the fused postprocess kernel
|
||||
# only when that kernel will actually run. The kernel is
|
||||
# gated on spec-decode + hybrid (see MambaBuffers.create);
|
||||
# without it, ``mamba_bufs.postprocess_align`` is None and
|
||||
# the staging buffers don't exist.
|
||||
if mamba_bufs.postprocess_align is not None:
|
||||
mamba_utils.stage_postprocess_inputs_to_gpu(
|
||||
mamba_bufs.postprocess_align,
|
||||
scheduler_output,
|
||||
self.input_batch.req_ids,
|
||||
num_reqs,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
)
|
||||
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
@@ -7114,7 +7142,7 @@ class GPUModelRunner(
|
||||
"""
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._mamba_copy_bufs = None
|
||||
self._mamba_bufs = None
|
||||
self.may_add_encoder_only_layers_to_kv_cache_config()
|
||||
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
|
||||
self.initialize_attn_backend(kv_cache_config, is_profiling=is_profiling)
|
||||
|
||||
Reference in New Issue
Block a user