[Bugfix][Core] Reserve the KV null block when validating max_model_len (#47272)
Signed-off-by: hyungjun oh <92hyungjun@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
c0233fcf01
commit
76fb6d210a
@@ -332,11 +332,12 @@ def run_test(
|
||||
):
|
||||
spec_decoding = spec_config is not None
|
||||
cache_arg: dict[str, Any] = (
|
||||
# Force preemptions: with 32 blocks the cache holds at most a single
|
||||
# max-length request, so the ~34 concurrent prompts contend and trigger
|
||||
# preemption. (Prompts here are << max_model_len, so dropping
|
||||
# max_model_len from 4096 to 512 doesn't change generation behavior.)
|
||||
dict(num_gpu_blocks_override=32, max_model_len=512)
|
||||
# Force preemptions: with 33 blocks (one is the reserved null block)
|
||||
# the cache holds at most a single max-length request, so the ~34
|
||||
# concurrent prompts contend and trigger preemption. (Prompts here are
|
||||
# << max_model_len, so dropping max_model_len from 4096 to 512 doesn't
|
||||
# change generation behavior.)
|
||||
dict(num_gpu_blocks_override=33, max_model_len=512)
|
||||
if test_preemption
|
||||
else dict(gpu_memory_utilization=0.9, max_model_len=4096)
|
||||
)
|
||||
|
||||
@@ -77,8 +77,12 @@ def test_auto_fit_max_model_len_rejects_oversized_input(
|
||||
|
||||
# Use a small KV cache budget to force auto-fit to a small
|
||||
# max_model_len. Pin block_size=16 so the budget is independent
|
||||
# of the platform's default block size.
|
||||
kv_cache_bytes = 1_000_000 # 1 MB
|
||||
# of the platform's default block size. One block for this model is
|
||||
# 2 (K/V) * 16 (block) * 12 (heads) * 64 (head_dim) * 2 (fp16)
|
||||
# * 12 (layers) = 589,824 bytes, and the pool must cover at least
|
||||
# two: one is reserved as the null block, so a 1 MB single-block
|
||||
# pool cannot serve any tokens at all.
|
||||
kv_cache_bytes = 2_000_000 # 2 MB = 3 blocks -> 2 usable
|
||||
|
||||
with vllm_runner(
|
||||
model_name=model,
|
||||
|
||||
Reference in New Issue
Block a user