[3/N][KV-Cache Layout Refactor] Standardize Mamba cache; drop get_transfer_cache_regions (#44456)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
parent
eb44b3aaa4
commit
6700813f86
@@ -126,7 +126,6 @@ from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
current_stream,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
@@ -7381,27 +7380,15 @@ class GPUModelRunner(
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
has_mamba = True
|
||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
state_tensors = []
|
||||
storage_offset_bytes = 0
|
||||
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
|
||||
dtype_size = get_dtype_size(dtype)
|
||||
num_element_per_page = (
|
||||
kv_cache_spec.page_size_bytes // dtype_size
|
||||
)
|
||||
target_shape = (num_blocks, *shape)
|
||||
stride = torch.empty(target_shape).stride()
|
||||
target_stride = (num_element_per_page, *stride[1:])
|
||||
assert storage_offset_bytes % dtype_size == 0
|
||||
tensor = torch.as_strided(
|
||||
raw_tensor.view(dtype),
|
||||
size=target_shape,
|
||||
stride=target_stride,
|
||||
storage_offset=storage_offset_bytes // dtype_size,
|
||||
)
|
||||
state_tensors.append(tensor)
|
||||
storage_offset_bytes += stride[0] * dtype_size
|
||||
|
||||
kv_caches[layer_name] = state_tensors
|
||||
page_size_bytes = kv_cache_spec.page_size_bytes
|
||||
# Hold a single contiguous [num_blocks, 1, 1, page_size_bytes]
|
||||
# int8 page view per layer; the layer's bind_kv_cache unpacks
|
||||
# each block's bytes into its conv/ssm state views. Keeping
|
||||
# one tensor per layer lets the KV connector register it
|
||||
# without special-casing Mamba.
|
||||
kv_caches[layer_name] = raw_tensor[
|
||||
: num_blocks * page_size_bytes
|
||||
].view(num_blocks, 1, 1, page_size_bytes)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
Reference in New Issue
Block a user