[Bugfix] Mamba CPU Offloading (#44599)
Signed-off-by: varun sundar rabindranath <vsundarr@redhat.com> Co-authored-by: varun sundar rabindranath <vsundarr@redhat.com>
This commit is contained in:
co-authored by
varun sundar rabindranath
parent
e0b9fb1290
commit
b927004c44
@@ -554,3 +554,91 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
finally:
|
||||
subscriber.close()
|
||||
del llm
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="HMA mamba-align CPU offload test is CUDA-only",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model,block_size,tp_size",
|
||||
[
|
||||
# ("Qwen/Qwen3.6-35B-A3B", 1056, 2),
|
||||
# ("tiiuae/falcon-mamba-7b", 16, 1),
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1)
|
||||
],
|
||||
)
|
||||
def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"cpu_bytes_to_use": 4 << 30,
|
||||
"block_size": block_size,
|
||||
},
|
||||
)
|
||||
llm = LLM(
|
||||
model=model,
|
||||
max_model_len=block_size * 10,
|
||||
gpu_memory_utilization=0.85,
|
||||
tensor_parallel_size=tp_size,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
|
||||
_PROMPT_SIZE: int = block_size * 2
|
||||
_PROMPT_TEXT = "Hi. Give me a set of trivia questions and their answers "
|
||||
|
||||
# build prompt ids to match prompt_size
|
||||
tokenizer = llm.get_tokenizer()
|
||||
raw_ids: list[int] = tokenizer.encode(_PROMPT_TEXT)
|
||||
while len(raw_ids) < _PROMPT_SIZE:
|
||||
raw_ids = tokenizer.encode("....") + raw_ids
|
||||
initial_ids: list[int] = raw_ids[:_PROMPT_SIZE]
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=128, temperature=0, ignore_eos=True)
|
||||
|
||||
failures: list[str] = []
|
||||
|
||||
def _get_output_str(outputs):
|
||||
return outputs[0].outputs[0].text
|
||||
|
||||
def _verify(llm, prompt, label: str):
|
||||
cold_outputs = llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
_wait_for_prefix_cache_reset(llm)
|
||||
cpu_outputs = llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
|
||||
cold_text = _get_output_str(cold_outputs)
|
||||
cpu_text = _get_output_str(cpu_outputs)
|
||||
print(f"{label} : cold outputs\n{cold_text}")
|
||||
print(f"{label} : cpu outputs\n{cpu_text}")
|
||||
|
||||
if cold_text != cpu_text:
|
||||
failures.append(
|
||||
f"{label}: mismatch\n cold: {cold_text!r}\n cpu: {cpu_text!r}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Mamba has only a single state. The CPU cache stores are triggered
|
||||
# at offload block boundaries. When the prompt is exactly at the boundary,
|
||||
# The CPU offload should not load the cached block.
|
||||
# This is because we'd use that state to recompute the last token. This
|
||||
# does not work for mamba as there is only one KV value and that is for
|
||||
# for the token at the boundary.
|
||||
# This is fine for other attention types as we have all the necessary
|
||||
# token KV values in the hit blocks.
|
||||
prompt = TokensPrompt(prompt_token_ids=initial_ids)
|
||||
_verify(llm, prompt, "block-boundary-prompt")
|
||||
|
||||
# Test for prompt token ids at non-block boundaries.
|
||||
# Reuse is okay for this case.
|
||||
prompt = TokensPrompt(prompt_token_ids=[0] + initial_ids)
|
||||
_verify(llm, prompt, "block-mid-prompt")
|
||||
|
||||
assert not failures, "\n\n".join(failures)
|
||||
|
||||
finally:
|
||||
del llm
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -94,6 +94,24 @@ def get_sliding_window_size_in_blocks(
|
||||
return None
|
||||
|
||||
|
||||
def resolve_mamba_align_size(spec: "OffloadingSpec") -> int | None:
|
||||
"""Scan all KV cache groups in *spec* and return the single mamba alignment
|
||||
size, or None if no group requires mamba alignment.
|
||||
|
||||
For MambaSpec groups in "align" cache mode the hit window must be rounded
|
||||
down to a multiple of the offloaded block size. Asserts that all such
|
||||
groups agree on the same value.
|
||||
"""
|
||||
mamba_align_size: int | None = None
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
|
||||
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
|
||||
offload_block_size = gpu_block_size * spec.block_size_factor
|
||||
assert mamba_align_size is None or mamba_align_size == offload_block_size
|
||||
mamba_align_size = offload_block_size
|
||||
return mamba_align_size
|
||||
|
||||
|
||||
class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_group_configs: tuple[GroupOffloadConfig, ...]
|
||||
block_size_factor: int
|
||||
@@ -290,6 +308,7 @@ class OffloadingConnectorScheduler:
|
||||
# used by _lookup
|
||||
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
|
||||
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
|
||||
self._mamba_align_size: int | None = resolve_mamba_align_size(spec)
|
||||
|
||||
self._req_status: dict[ReqId, RequestOffloadState] = {}
|
||||
self._current_batch_load_jobs: dict[int, TransferJob] = {}
|
||||
@@ -408,6 +427,12 @@ class OffloadingConnectorScheduler:
|
||||
# for sliding window attention, we must reduce by 1 to make sure
|
||||
# we still have a hit after reduction
|
||||
max_hit_size_tokens -= 1
|
||||
if self._mamba_align_size is not None:
|
||||
# Constrain hit-window to the mamba block size.
|
||||
max_hit_size_tokens = round_down(
|
||||
max_hit_size_tokens, self._mamba_align_size
|
||||
)
|
||||
|
||||
num_hit_tokens: int = 0
|
||||
defer_lookup = False
|
||||
lookup_groups = self._lookup_groups
|
||||
|
||||
Reference in New Issue
Block a user