diff --git a/csrc/cumem_allocator.cpp b/csrc/cumem_allocator.cpp index 2329d51a14..720c990450 100644 --- a/csrc/cumem_allocator.cpp +++ b/csrc/cumem_allocator.cpp @@ -212,6 +212,29 @@ void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem, if (error_code != 0) { return; } + +#ifdef USE_ROCM + // ROCm-only: zero the freshly (re)mapped region. + // + // On some amdgpu driver versions the physical VRAM backing a buffer is not + // scrubbed on release, so the pages handed back here (e.g. on wake_up after a + // sleep that discarded a tag) can still contain stale data from a previously + // freed allocation -- possibly from another process. Callers of the device + // allocator expect newly mapped memory to be well defined; in particular + // hybrid Mamba / gated-delta-net (GDN) models keep a persisted conv/recurrent + // state cache, and a new sequence that reads an unzeroed slot will propagate + // garbage/NaN, collapsing generation to token id 0 (decoded as "!"). + // + // This is scoped to the ROCm path on purpose: the CUDA path is unchanged. + // Use CUDA_CHECK so a failure sets the global error_code; the Python caller + // (python_create_and_map) only inspects error_code, so returning without + // setting it would make a failed memset look successful and still hand back + // unzeroed memory. + CUDA_CHECK((CUresult)hipMemset(reinterpret_cast(d_mem), 0, size)); + if (error_code != 0) { + return; + } +#endif // std::cout << "create_and_map: device=" << device << ", size=" << size << ", // d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl; } diff --git a/tests/models/language/generation/test_gdn_sleep_wake.py b/tests/models/language/generation/test_gdn_sleep_wake.py new file mode 100644 index 0000000000..e61c274aec --- /dev/null +++ b/tests/models/language/generation/test_gdn_sleep_wake.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test for hybrid GDN/Mamba models under sleep -> wake. + +Hybrid Mamba / gated-delta-net (GDN) models (e.g. Qwen3-Next) keep a +persisted conv + recurrent state cache. With sleep mode (the RLHF reuse +pattern: ``sleep()`` -> weight update -> ``wake_up()``) the state-cache tag is +discarded on sleep and its device memory is re-created on wake. If a *new* +sequence's state slot is consumed before being reset, the gated-delta-rule +kernel faithfully propagates whatever is in that (now non-zeroed) memory; when +it contains NaN/inf the output becomes NaN and ``argmax`` collapses every token +to id 0 (which decodes to ``"!"``), giving ``reward=0`` / NaN log-probs in RL +training. + +This test sleeps and wakes a small hybrid GDN model and asserts that +post-wake generation is neither degenerate (single repeated token) nor NaN. +""" + +import pytest + +from vllm import LLM, SamplingParams + +# Small Qwen3-Next (GDN) model already used by the hybrid model test-suite. +MODEL = "tiny-random/qwen3-next-moe" + +PROMPTS = [ + "The capital of France is", + "Once upon a time,", + "1, 2, 3, 4,", + "Water is made of", +] + + +@pytest.mark.hybrid_model +def test_gdn_sleep_wake_no_stale_state(): + sampling_params = SamplingParams(temperature=0.0, max_tokens=32, logprobs=1) + + # Keep the reserved fraction low. On some (notably ROCm/amdgpu) drivers the + # VRAM discarded by ``sleep()`` is not returned to the free pool before + # ``wake_up()`` re-creates it, so the woken allocation must coexist with the + # not-yet-reclaimed one (~2x peak). A high ``gpu_memory_utilization`` then + # OOMs in ``cuMemCreate`` on wake. The model is tiny, so a small fraction + # still leaves ample KV/state cache while keeping the sleep/wake cycle well + # within device memory. + llm = LLM( + model=MODEL, + enable_sleep_mode=True, + enforce_eager=True, + max_model_len=1024, + gpu_memory_utilization=0.4, + trust_remote_code=True, + ) + + # Warm generation before sleeping. + llm.generate(PROMPTS, sampling_params) + + # Default sleep offloads weights and DISCARDS the kv / GDN state cache; + # wake_up re-creates that memory (fresh, not guaranteed zeroed). + llm.sleep() + llm.wake_up() + + after = llm.generate(PROMPTS, sampling_params) + + for output in after: + completion = output.outputs[0] + token_ids = list(completion.token_ids) + assert token_ids, "empty generation after wake_up" + # The bug collapses every token to a single id (e.g. 0 -> "!"). + assert len(set(token_ids)) > 1, ( + f"degenerate single-token output after wake_up: {token_ids[:16]}" + ) + # NaN logits surface as NaN log-probs. + for step_logprobs in completion.logprobs or []: + for logprob in step_logprobs.values(): + assert logprob.logprob == logprob.logprob, ( + "NaN log-prob after wake_up (stale GDN state)" + )