Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -216,15 +216,6 @@ def test_models_distributed(
|
||||
pytest.skip(f"Skip test for {TARGET_TEST_SUITE}")
|
||||
|
||||
with monkeypatch.context() as monkeypatch_context:
|
||||
if (
|
||||
model == "meta-llama/Llama-3.2-1B-Instruct"
|
||||
and distributed_executor_backend == "ray"
|
||||
and attention_backend == ""
|
||||
and TARGET_TEST_SUITE == "L4"
|
||||
and enable_prompt_embeds
|
||||
): # noqa
|
||||
pytest.skip("enable_prompt_embeds does not work with ray compiled dag.")
|
||||
|
||||
for k, v in extra_env.items():
|
||||
monkeypatch_context.setenv(k, v)
|
||||
|
||||
|
||||
@@ -157,10 +157,17 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
)
|
||||
|
||||
if self.kv_cache_dtype != self.dtype:
|
||||
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_float = 1.0
|
||||
self.attn._v_scale_float = 1.0
|
||||
# Non-unit scales so the fused and unfused writers must agree on them.
|
||||
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_cpu = torch.tensor(
|
||||
0.5, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._v_scale_cpu = torch.tensor(
|
||||
0.25, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._k_scale_float = 0.5
|
||||
self.attn._v_scale_float = 0.25
|
||||
else:
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
@@ -527,16 +534,6 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
else:
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
|
||||
# The AITER PTS kernel populates k_out only for non-FP8 caches.
|
||||
# With FP8, the kernel writes quantized K directly to the cache
|
||||
# and may leave k_out uninitialised. In production this is fine
|
||||
# because downstream attention reads K from the cache.
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
|
||||
# Allow one E4M3 quantization step while still comparing every block.
|
||||
if is_fp8_cache:
|
||||
@@ -544,6 +541,18 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
else:
|
||||
cache_atol, cache_rtol = ATOL, RTOL
|
||||
|
||||
# The fused k_out is the dequantized cache write, so it carries the same
|
||||
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
|
||||
# from the cache and asks the kernel not to write k_out at all.
|
||||
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
|
||||
torch.testing.assert_close(
|
||||
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
|
||||
)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.float(),
|
||||
kv_cache_fused.float(),
|
||||
@@ -570,20 +579,52 @@ _FUSION_CONFIGS = [
|
||||
_FUSION_CONFIGS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
"attn_backend, use_shuffle_kv_layout, kv_layout",
|
||||
[
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
|
||||
# K/V side contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="rocm_attn",
|
||||
),
|
||||
# Unified never reads the shuffle env; its strided views take any layout.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="unified-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="unified-token_major",
|
||||
),
|
||||
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
|
||||
# contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="fa-token_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"1",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-shuffle",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [5, 2048])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_layout",
|
||||
[
|
||||
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
|
||||
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@@ -615,6 +656,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
use_shuffle_kv_layout: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
@@ -627,7 +669,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
rotary_dim=rotary_dim,
|
||||
block_size=block_size,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout="0",
|
||||
use_shuffle_kv_layout=use_shuffle_kv_layout,
|
||||
kv_layout=kv_layout,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cpu_all_reduce_ (gloo, CPU only)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.distributed.utils import (
|
||||
cpu_all_reduce_,
|
||||
stateless_init_torch_distributed_process_group,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _make_input(rank: int, world_size: int, step: int) -> torch.Tensor:
|
||||
# Same shape as the V2 DP metadata tensor; values differ per rank/step.
|
||||
t = torch.zeros(6, world_size, dtype=torch.int32)
|
||||
t[:, rank] = torch.arange(6, dtype=torch.int32) * (rank + 1) + step
|
||||
return t
|
||||
|
||||
|
||||
def _worker(rank: int, world_size: int, port: int, stateless_port: int):
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
init_method=f"tcp://127.0.0.1:{port}",
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
coord = GroupCoordinator(
|
||||
group_ranks=[list(range(world_size))],
|
||||
local_rank=0,
|
||||
torch_distributed_backend="gloo",
|
||||
use_device_communicator=False,
|
||||
group_name="cpu_all_reduce_test",
|
||||
)
|
||||
|
||||
# Several steps to exercise tag reuse across consecutive calls.
|
||||
for step in range(5):
|
||||
expected = _make_input(rank, world_size, step)
|
||||
dist.all_reduce(expected, group=coord.cpu_group)
|
||||
t = _make_input(rank, world_size, step)
|
||||
out = coord.cpu_all_reduce_(t)
|
||||
assert out is t
|
||||
torch.testing.assert_close(t, expected, rtol=0, atol=0)
|
||||
|
||||
# Float sums must be bitwise identical on every rank.
|
||||
t = torch.randn(6, world_size, generator=torch.Generator().manual_seed(rank))
|
||||
expected = t.clone()
|
||||
dist.all_reduce(expected, group=coord.cpu_group)
|
||||
coord.cpu_all_reduce_(t)
|
||||
torch.testing.assert_close(t, expected)
|
||||
gathered = [torch.empty_like(t) for _ in range(world_size)]
|
||||
dist.all_gather(gathered, t, group=coord.cpu_group)
|
||||
for other in gathered:
|
||||
assert torch.equal(other, t)
|
||||
|
||||
# Unregistered (stateless) process group, as used by the engine-core DP
|
||||
# group and elastic EP.
|
||||
stateless_pg = stateless_init_torch_distributed_process_group(
|
||||
"127.0.0.1", stateless_port, rank, world_size, backend="gloo"
|
||||
)
|
||||
expected = _make_input(rank, world_size, 0)
|
||||
dist.all_reduce(expected)
|
||||
t = _make_input(rank, world_size, 0)
|
||||
cpu_all_reduce_(t, stateless_pg)
|
||||
torch.testing.assert_close(t, expected, rtol=0, atol=0)
|
||||
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
# Non-power-of-two sizes exercise the cross-block exchange.
|
||||
@pytest.mark.parametrize("world_size", [2, 3, 4, 6, 8, 9, 12])
|
||||
def test_cpu_all_reduce(world_size: int):
|
||||
mp.spawn(
|
||||
_worker,
|
||||
args=(world_size, get_open_port(), get_open_port()),
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
)
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.plugins_tests.test_oot_registration_online import (
|
||||
run_and_test_dummy_opt_api_server,
|
||||
)
|
||||
@@ -8,3 +12,17 @@ from tests.plugins_tests.test_oot_registration_online import (
|
||||
|
||||
def test_distributed_oot(dummy_opt_path: str):
|
||||
run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
def test_distributed_oot_pp_raw_input_tokens(dummy_opt_path: str, enforce_eager: bool):
|
||||
# The model asserts that input_ids reaches every pipeline stage.
|
||||
overrides = {"architectures": ["MyOPTRawInputTokensForCausalLM"]}
|
||||
extra_args = ["-pp", "2", "--hf-overrides", json.dumps(overrides)]
|
||||
if enforce_eager:
|
||||
extra_args.append("--enforce-eager")
|
||||
run_and_test_dummy_opt_api_server(
|
||||
dummy_opt_path,
|
||||
extra_args=extra_args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
)
|
||||
|
||||
@@ -51,7 +51,7 @@ def _traffic_loop(
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None]]:
|
||||
) -> list[tuple[float, float, int | None, str | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
@@ -60,15 +60,18 @@ def _traffic_loop(
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
phase = None
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
if is_probe and status_code == 200:
|
||||
phase = response.json()["phase"]
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code))
|
||||
if status_code == 200:
|
||||
responses.append((request_start, time.perf_counter(), status_code, phase))
|
||||
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
@@ -78,14 +81,16 @@ def _traffic_loop(
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
|
||||
rejected = [end for _, end, status in responses if status == 503]
|
||||
if not rejected:
|
||||
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
|
||||
committing = [end for _, end, _, phase in responses if phase == "committing"]
|
||||
if not committing:
|
||||
return 0
|
||||
recovered = next(
|
||||
end for _, end, status in responses if status == 200 and end > rejected[-1]
|
||||
end
|
||||
for _, end, status, phase in responses
|
||||
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
|
||||
)
|
||||
return recovered - rejected[0]
|
||||
return recovered - committing[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
@@ -124,16 +129,20 @@ def _scale_with_traffic(
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status in responses
|
||||
for _, _, status, _ in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_503 = [start for start, _, status in probe_result if status == 503]
|
||||
assert probe_503, "Scaling probe did not observe commit"
|
||||
probe_committing = [
|
||||
start for start, _, _, phase in probe_result if phase == "committing"
|
||||
]
|
||||
assert probe_committing, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200 and start_time <= request_start and request_end < probe_503[0]
|
||||
status == 200
|
||||
and start_time <= request_start
|
||||
and request_end < probe_committing[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status in responses
|
||||
for request_start, request_end, status, _ in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
|
||||
@@ -585,6 +585,7 @@ def _check_dummy_hash_model_forward(
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.parallel_config = vllm_config.parallel_config
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Make sure ray assigns GPU workers to the correct node.
|
||||
|
||||
Run:
|
||||
```sh
|
||||
cd $VLLM_PATH/tests
|
||||
|
||||
pytest distributed/test_multi_node_assignment.py
|
||||
```
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
from vllm import initialize_ray_cluster
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.utils.network_utils import get_ip
|
||||
from vllm.v1.executor.ray_utils import available_resources_per_node
|
||||
|
||||
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not VLLM_MULTI_NODE, reason="Need at least 2 nodes to run the test."
|
||||
)
|
||||
def test_multi_node_assignment() -> None:
|
||||
# NOTE: important to keep this class definition here
|
||||
# to let ray use cloudpickle to serialize it.
|
||||
class Actor:
|
||||
def get_ip(self):
|
||||
return get_ip()
|
||||
|
||||
for _ in range(10):
|
||||
config = ParallelConfig(1, 2)
|
||||
initialize_ray_cluster(config)
|
||||
|
||||
current_ip = get_ip()
|
||||
workers = []
|
||||
for bundle_id, bundle in enumerate(config.placement_group.bundle_specs):
|
||||
if not bundle.get("GPU", 0):
|
||||
continue
|
||||
scheduling_strategy = PlacementGroupSchedulingStrategy(
|
||||
placement_group=config.placement_group,
|
||||
placement_group_capture_child_tasks=True,
|
||||
placement_group_bundle_index=bundle_id,
|
||||
)
|
||||
|
||||
worker = ray.remote(
|
||||
num_cpus=0,
|
||||
num_gpus=1,
|
||||
scheduling_strategy=scheduling_strategy,
|
||||
)(Actor).remote()
|
||||
worker_ip = ray.get(worker.get_ip.remote())
|
||||
assert worker_ip == current_ip
|
||||
workers.append(worker)
|
||||
|
||||
for worker in workers:
|
||||
ray.kill(worker)
|
||||
|
||||
ray.util.remove_placement_group(config.placement_group)
|
||||
|
||||
gpus_needed = len(workers)
|
||||
deadline = time.monotonic() + 60
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
while True:
|
||||
free_gpus = available_resources_per_node().get(node_id, {}).get("GPU", 0)
|
||||
if free_gpus >= gpus_needed:
|
||||
break
|
||||
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError("placement group resources were not released")
|
||||
|
||||
time.sleep(min(1, remaining))
|
||||
@@ -598,6 +598,21 @@ class TestCohereErrorEnvelope:
|
||||
assert "message" in body
|
||||
assert isinstance(body["message"], str) and body["message"]
|
||||
|
||||
@pytest.mark.parametrize("path", ["/cohere/v2/chat", "/cohere/v2/chat/render"])
|
||||
def test_required_tool_choice_without_tools_returns_400(self, path):
|
||||
"""Reject an impossible tool choice before reaching either handler."""
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
path,
|
||||
json={**_minimal_request_body(), "tool_choice": "REQUIRED"},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = response.json()
|
||||
assert "error" not in body
|
||||
assert "REQUIRED" in body["message"]
|
||||
assert "tools" in body["message"]
|
||||
|
||||
def test_http_error_body_is_cohere_shaped(self):
|
||||
# A raised ``HTTPException`` from anywhere in the request cycle
|
||||
# is routed through ``http_exception_handler`` (producing the
|
||||
|
||||
@@ -95,6 +95,27 @@ class TestCohereChatV2Request:
|
||||
tool_choice="ANY", # not REQUIRED/NONE
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("tool_fields", [{}, {"tools": None}, {"tools": []}])
|
||||
def test_required_tool_choice_requires_tools(self, tool_fields):
|
||||
with pytest.raises(VLLMValidationError, match="REQUIRED.*tools") as exc_info:
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice="REQUIRED",
|
||||
**tool_fields,
|
||||
)
|
||||
assert exc_info.value.parameter == "tool_choice"
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", [None, "NONE"])
|
||||
def test_nonrequired_tool_choice_allows_empty_tools(self, tool_choice):
|
||||
request = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice=tool_choice,
|
||||
tools=[],
|
||||
)
|
||||
assert request.tool_choice == tool_choice
|
||||
|
||||
def test_stop_sequences_bounded_by_env_limit(self):
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
limit = envs.VLLM_MAX_STOP_STRINGS
|
||||
|
||||
@@ -64,6 +64,7 @@ def test_load_prompt_embeds(
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.enable_prompt_embeds = True
|
||||
model_config.get_hidden_size.return_value = hidden_size
|
||||
model_config.hf_text_config = object()
|
||||
model_config.dtype = dtype
|
||||
|
||||
# construct arbitrary tensors of various dtypes, layouts, and sizes.
|
||||
|
||||
@@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix(
|
||||
assert choices[0].finish_reason == "length"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("stop", "include_stop_str_in_output", "dropped"),
|
||||
[
|
||||
("eos", False, True),
|
||||
("stop_token_id", False, True),
|
||||
("stop_string", False, False),
|
||||
("eos", True, False),
|
||||
],
|
||||
)
|
||||
async def test_parsed_derender_leaves_out_stop_token_like_engine(
|
||||
tokenizer, stop, include_stop_str_in_output, dropped
|
||||
):
|
||||
"""A parser keeps special tokens in the decoded text, so the token that
|
||||
stopped generation must be dropped the way the engine drops it."""
|
||||
eos = tokenizer.eos_token_id
|
||||
answer_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
bang = tokenizer.encode("!", add_special_tokens=False)[-1]
|
||||
last_id, stop_reason = {
|
||||
"eos": (eos, None),
|
||||
"stop_token_id": (bang, bang),
|
||||
"stop_string": (bang, "!"),
|
||||
}[stop]
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, [])
|
||||
parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage(
|
||||
content=text
|
||||
)
|
||||
request = _chat_request(include_stop_str_in_output=include_stop_str_in_output)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
renderer.get_eos_token_id.return_value = eos
|
||||
renderer._executor = executor
|
||||
model_config = MagicMock()
|
||||
model_config.hf_config.model_type = "llama"
|
||||
model_config.model = MODEL_NAME
|
||||
model_config.try_get_generation_config.return_value = {}
|
||||
derenderer = OnlineDerenderer(
|
||||
model_config=model_config,
|
||||
renderer=renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="string",
|
||||
)
|
||||
derenderer.parser = MagicMock(return_value=parser)
|
||||
|
||||
def batch_content(token_ids: list[int], finish_reason: str) -> str:
|
||||
choice = GenerateTokensChoice(
|
||||
index=0,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason if finish_reason == "stop" else None,
|
||||
)
|
||||
response = GenerateTokensResponse(
|
||||
request_id="test", prompt_token_ids=[11, 12], choices=[choice]
|
||||
)
|
||||
return derenderer._derender_chat(response, request)[0].message.content
|
||||
|
||||
# Same decode window and seeding; "length" never drops a token.
|
||||
expected = batch_content(
|
||||
answer_ids if dropped else answer_ids + [last_id], "length"
|
||||
)
|
||||
batch = batch_content(answer_ids + [last_id], "stop")
|
||||
stream, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason
|
||||
),
|
||||
chat_request=request,
|
||||
prompt_token_ids=[11, 12],
|
||||
)
|
||||
|
||||
assert batch == expected
|
||||
assert stream.choices[0].delta.content == expected
|
||||
|
||||
|
||||
class TestDetokenizeDelta:
|
||||
"""_detokenize_delta: chunked decode must equal one shot decode."""
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ def test_sentencepiece_leading_space_matches_engine():
|
||||
assert [t.token for t in entry.top_logprobs] == convert_ids_list_to_tokens(
|
||||
tokenizer, [spaced, bare]
|
||||
)
|
||||
completion = _convert_chat_logprobs_to_completion_logprobs(resolved)
|
||||
completion = _convert_chat_logprobs_to_completion_logprobs(resolved, logprobs)
|
||||
assert completion.top_logprobs == [{" true": -0.5, "true": -1.5}]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""generate + derender must return the same `top_logprobs` as the coupled
|
||||
`/v1/chat/completions` and `/v1/completions` endpoints (#59513).
|
||||
|
||||
Both sides start from the same per-position engine dicts, built with the
|
||||
engine's own `append_logprobs_for_next_position` and detokenized the way the
|
||||
engine does, and go through the real coupled serving objects on one side and
|
||||
`/inference/v1/generate` + derender on the other. The cases include a sampled
|
||||
token outside the top k, where chat keeps k entries and completions k + 1; the
|
||||
greedy end-to-end parity tests never produce that case.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
_build_mock_engine,
|
||||
_build_serving_chat,
|
||||
)
|
||||
from tests.entrypoints.openai.completion.test_completion_error import (
|
||||
_build_serving_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProbs
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import append_logprobs_for_next_position
|
||||
from vllm.renderers.online_derenderer import (
|
||||
_chat_top_logprobs_limit,
|
||||
_completion_top_logprobs_limit,
|
||||
_convert_chat_logprobs_to_completion_logprobs,
|
||||
_resolve_logprobs,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return get_tokenizer(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def serving_chat():
|
||||
return _build_serving_chat(_build_mock_engine())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def serving_completion():
|
||||
return _build_serving_completion(_build_mock_engine())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vocab_ids(tokenizer):
|
||||
"""Ordinary word pieces (no byte-fallback tokens, which need the U+FFFD
|
||||
context repair that is compared elsewhere)."""
|
||||
ids = tokenizer.encode(
|
||||
"the quick brown fox jumps over a lazy dog while seven cats watch "
|
||||
"from an old wooden fence near the river bank",
|
||||
add_special_tokens=False,
|
||||
)
|
||||
pieces = tokenizer.convert_ids_to_tokens(ids)
|
||||
return list(dict.fromkeys(i for i, p in zip(ids, pieces) if "<0x" not in p))
|
||||
|
||||
|
||||
def _positions(tokenizer, vocab_ids, k, sampled_ranks):
|
||||
"""Engine output for one completion: per position the sampled id and its
|
||||
dict (sampled first, then ranks 1..k), with decoded tokens as the engine
|
||||
fills them. k = -1 returns every candidate, as for the whole vocabulary."""
|
||||
token_ids, dicts = [], []
|
||||
for pos, sampled_rank in enumerate(sampled_ranks):
|
||||
n_top = max(k, sampled_rank) if k != -1 else len(vocab_ids) - 1
|
||||
ranked = [vocab_ids[(pos + j) % len(vocab_ids)] for j in range(n_top + 1)]
|
||||
sampled = ranked[sampled_rank - 1]
|
||||
top = ranked[: len(ranked) - 1] if k == -1 else ranked[:k]
|
||||
ids = [sampled, *top]
|
||||
logprobs = [-0.25 * sampled_rank] + [-0.25 * (r + 1) for r in range(len(top))]
|
||||
per_position: list = []
|
||||
append_logprobs_for_next_position(
|
||||
per_position,
|
||||
ids,
|
||||
logprobs,
|
||||
convert_ids_list_to_tokens(tokenizer, ids),
|
||||
sampled_rank,
|
||||
len(top),
|
||||
)
|
||||
token_ids.append(sampled)
|
||||
dicts.append(per_position[0])
|
||||
return token_ids, dicts
|
||||
|
||||
|
||||
def _generate(token_ids, dicts) -> GenerateLogProbs:
|
||||
out = ServingTokens._create_tokens_logprobs(
|
||||
None, token_ids=token_ids, top_logprobs=dicts
|
||||
)
|
||||
# Over the wire, as a derender client would send it back.
|
||||
return GenerateLogProbs.model_validate_json(out.model_dump_json())
|
||||
|
||||
|
||||
# Sampled token rank per position: top-1, inside the top k, outside it.
|
||||
RANKS = [1, 2, 1, 7, 3, 9, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
|
||||
def test_chat_derender_matches_coupled_chat(tokenizer, vocab_ids, serving_chat, k):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
coupled = serving_chat._create_chat_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
tokenizer=tokenizer,
|
||||
num_output_top_logprobs=k,
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
logprobs=True,
|
||||
top_logprobs=k,
|
||||
)
|
||||
derendered = _resolve_logprobs(
|
||||
_generate(token_ids, dicts),
|
||||
tokenizer,
|
||||
top_limit=_chat_top_logprobs_limit(request),
|
||||
)
|
||||
assert derendered.model_dump() == coupled.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
|
||||
def test_completion_derender_matches_coupled_completion(
|
||||
tokenizer, vocab_ids, serving_completion, k
|
||||
):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
coupled = serving_completion._create_completion_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
num_output_top_logprobs=k,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
|
||||
generated = _generate(token_ids, dicts)
|
||||
derendered = _convert_chat_logprobs_to_completion_logprobs(
|
||||
_resolve_logprobs(
|
||||
generated,
|
||||
tokenizer,
|
||||
top_limit=_completion_top_logprobs_limit(request),
|
||||
),
|
||||
generated,
|
||||
)
|
||||
assert derendered.model_dump() == coupled.model_dump()
|
||||
if k > 0:
|
||||
# The case #59513 is about: completions keeps k + 1 entries when the
|
||||
# sampled token is outside the top k.
|
||||
outside = [i for i, r in enumerate(RANKS) if r > k]
|
||||
assert all(len(derendered.top_logprobs[i]) == k + 1 for i in outside)
|
||||
if k == -1:
|
||||
# /v1/completions returns every candidate for logprobs=-1 (#55717).
|
||||
assert [len(top) for top in derendered.top_logprobs] == [len(d) for d in dicts]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"top_limit",
|
||||
[
|
||||
_chat_top_logprobs_limit(None),
|
||||
# A request that does not ask for logprobs says nothing about the cut.
|
||||
_chat_top_logprobs_limit(
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME, messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
),
|
||||
_completion_top_logprobs_limit(None),
|
||||
_completion_top_logprobs_limit(
|
||||
CompletionRequest(model=MODEL_NAME, prompt="hi")
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_without_a_logprobs_request_derender_keeps_every_candidate(
|
||||
tokenizer, vocab_ids, top_limit
|
||||
):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, 2, RANKS)
|
||||
derendered = _resolve_logprobs(
|
||||
_generate(token_ids, dicts), tokenizer, top_limit=top_limit
|
||||
)
|
||||
assert [len(e.top_logprobs) for e in derendered.content] == [len(d) for d in dicts]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [-1, 0, 2])
|
||||
def test_completion_position_without_logprobs_stays_none(
|
||||
tokenizer, vocab_ids, serving_completion, k
|
||||
):
|
||||
"""A position the engine returned no logprobs for is `None` in
|
||||
`/v1/completions`, not `{}`; derender keeps it that way."""
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
dicts[3] = None
|
||||
coupled = serving_completion._create_completion_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
num_output_top_logprobs=k,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
|
||||
generated = _generate(token_ids, dicts)
|
||||
derendered = _convert_chat_logprobs_to_completion_logprobs(
|
||||
_resolve_logprobs(
|
||||
generated,
|
||||
tokenizer,
|
||||
top_limit=_completion_top_logprobs_limit(request),
|
||||
),
|
||||
generated,
|
||||
)
|
||||
assert derendered.top_logprobs == coupled.top_logprobs
|
||||
assert derendered.top_logprobs[3] is None
|
||||
@@ -14,11 +14,11 @@ from vllm.logprobs import Logprob
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own integer token id."""
|
||||
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
|
||||
num_output_top_logprobs=2,
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
|
||||
)
|
||||
token_ids = {e.token_id for e in result.content[0].top_logprobs}
|
||||
assert token_ids == {262, 257}, f"got {token_ids}"
|
||||
@@ -32,7 +32,6 @@ def test_sampled_entry_carries_token_id_and_rank():
|
||||
top_logprobs=[
|
||||
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 262
|
||||
@@ -50,7 +49,6 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
None,
|
||||
token_ids=[5],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 5
|
||||
@@ -60,14 +58,13 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
"""logprobs=0 must still emit 1 entry (the sampled token)."""
|
||||
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=0,
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 1
|
||||
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
|
||||
|
||||
|
||||
def test_logprobs_minus_one_emits_all_tokens():
|
||||
@@ -75,7 +72,6 @@ def test_logprobs_minus_one_emits_all_tokens():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=-1,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
@@ -86,7 +82,6 @@ def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token == "token_id:7"
|
||||
@@ -102,7 +97,6 @@ def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
@@ -121,7 +115,6 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
@@ -130,9 +123,9 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
|
||||
|
||||
def test_sampled_token_outside_topk_comes_first():
|
||||
"""The engine puts the sampled token first, then ranks 1..k. When the
|
||||
sampled token is outside the top k it takes one of the k slots, so rank k
|
||||
is left out (same as the OpenAI endpoints)."""
|
||||
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
|
||||
all k + 1 candidates when the sampled token is outside the top k; derender
|
||||
applies each endpoint's cut (#59513)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[50],
|
||||
@@ -143,9 +136,8 @@ def test_sampled_token_outside_topk_comes_first():
|
||||
20: Logprob(-1.0, rank=2),
|
||||
}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1]
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
|
||||
|
||||
|
||||
def test_logprob_is_required_on_the_wire():
|
||||
@@ -160,7 +152,6 @@ def test_rank_zero_from_a_nan_logprob_is_none():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.rank is None
|
||||
@@ -176,7 +167,6 @@ def test_text_logprobs_clamp_a_nan_logprob():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=MagicMock(),
|
||||
)
|
||||
entry = result.content[0]
|
||||
|
||||
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
|
||||
def test_paged_attention(
|
||||
kv_cache_factory,
|
||||
num_seqs: int,
|
||||
@@ -133,9 +134,14 @@ def test_paged_attention(
|
||||
kv_cache_dtype: str,
|
||||
seed: int,
|
||||
device: str,
|
||||
use_interleaved_v_cache: bool,
|
||||
) -> None:
|
||||
if current_platform.is_navi() and (
|
||||
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
|
||||
kv_cache_dtype == "fp8"
|
||||
or head_size != 128
|
||||
or block_size != 16
|
||||
or use_alibi
|
||||
or use_interleaved_v_cache
|
||||
):
|
||||
pytest.skip()
|
||||
|
||||
@@ -201,6 +207,20 @@ def test_paged_attention(
|
||||
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
|
||||
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
|
||||
|
||||
standard_value_cache = value_cache
|
||||
if use_interleaved_v_cache:
|
||||
# Hand the kernel V in the interleaved layout (same 4D shape); the
|
||||
# reference below gets the standard layout back.
|
||||
x_v = 16 // value_cache.element_size()
|
||||
value_cache = (
|
||||
value_cache.view(
|
||||
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
|
||||
)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Call the paged attention kernel.
|
||||
output = torch.empty_like(query)
|
||||
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
|
||||
@@ -234,6 +254,7 @@ def test_paged_attention(
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
@@ -259,10 +280,13 @@ def test_paged_attention(
|
||||
v_scale,
|
||||
None,
|
||||
"f16",
|
||||
use_interleaved_v_cache,
|
||||
),
|
||||
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
|
||||
)
|
||||
|
||||
value_cache = standard_value_cache
|
||||
|
||||
# Run the reference implementation.
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Convert cache data back to dtype.
|
||||
|
||||
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention(
|
||||
num_heads: int,
|
||||
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv(
|
||||
num_heads: int,
|
||||
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
|
||||
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(
|
||||
device: str, interleave_v: bool
|
||||
) -> None:
|
||||
# Boundary guard for the KV_FROM_CACHE block-table load. With an
|
||||
# exact-sized block table (row length == number of blocks for the
|
||||
# sequence) and a query that ends the sequence, the last K/V tile has
|
||||
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(num_blocks, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi(
|
||||
num_heads: int,
|
||||
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_f32(
|
||||
num_heads: int,
|
||||
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention(
|
||||
num_heads,
|
||||
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi_f32(
|
||||
num_heads: int,
|
||||
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention_alibi(
|
||||
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
|
||||
num_heads,
|
||||
num_queries_per_kv,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_qwen3_nonstandard_block_size(
|
||||
num_heads: int,
|
||||
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
|
||||
if not current_platform.is_rocm():
|
||||
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
|
||||
kv_cache_dtype="auto",
|
||||
device=device,
|
||||
op=op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
rtol=rtol,
|
||||
)
|
||||
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_asm_kv_scale_tables_shared_across_builders():
|
||||
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
|
||||
return the same dense table, a different scale a different one."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||
AiterFlashAttentionMetadataBuilder,
|
||||
AsmKvScaleTables,
|
||||
)
|
||||
|
||||
tables = AsmKvScaleTables()
|
||||
device = torch.device("cuda")
|
||||
table = tables.get(device, 8, 2, 16, 0.5)
|
||||
assert tables.get(device, 8, 2, 16, 0.5) is table
|
||||
assert tables.get(device, 8, 2, 16, 1.0) is not table
|
||||
assert table.shape == (8, 2, 16)
|
||||
assert table.dtype == torch.float32
|
||||
assert torch.all(table == 0.5)
|
||||
# The builder class holds the single process-wide instance.
|
||||
assert isinstance(
|
||||
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
|
||||
)
|
||||
|
||||
@@ -1989,8 +1989,8 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"""Replay V4.1 ROCm decode after device-only query reallocation."""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
@@ -2044,7 +2044,7 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
|
||||
mla_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_builder = DeepseekV41ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
assert (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.math_utils import next_power_of_2, round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
try:
|
||||
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
|
||||
|
||||
def _make_mxfp8_moe_weights(
|
||||
e: int, n: int, k: int
|
||||
e: int, n: int, k: int, is_gated: bool = True
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, s = zip(
|
||||
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
|
||||
)
|
||||
return torch.stack(q), torch.stack(s)
|
||||
|
||||
w1_rows = 2 * n if is_gated else n
|
||||
w1, w1_scale = quantize(
|
||||
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
|
||||
@@ -806,16 +808,20 @@ def _make_mxfp8_moe_weights(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
|
||||
@pytest.mark.parametrize("transposed", [True, False], ids=["transposed", "row-major"])
|
||||
@pytest.mark.parametrize("m", [1, 16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
m: int, block_shape: list[int], workspace_init
|
||||
m: int, block_shape: list[int], transposed: bool, workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
|
||||
finalize to the caller."""
|
||||
finalize to the caller. For fp8-block, ``transposed`` toggles the oracle's
|
||||
is_scale_transposed column-major activation-scale layout."""
|
||||
if block_shape != [128, 128] and transposed:
|
||||
pytest.skip("column-major scales apply to fp8 [128, 128] block only")
|
||||
e, topk, n, k = 32, 4, 1024, 1024
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
@@ -849,6 +855,9 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
is_block = block_shape == [128, 128]
|
||||
if is_block and not transposed:
|
||||
quant_config = replace(quant_config, is_scale_transposed=False)
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
@@ -866,19 +875,27 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config, quant_config=quant_config, allow_new_interface=True
|
||||
),
|
||||
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
def make_kernel(quant_config: FusedMoEQuantConfig) -> mk.FusedMoEKernel:
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
),
|
||||
TrtLlmFp8ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
),
|
||||
)
|
||||
|
||||
kernel = make_kernel(quant_config)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
|
||||
def apply(k_: mk.FusedMoEKernel) -> torch.Tensor:
|
||||
return k_.apply(
|
||||
hidden_states=a,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
@@ -888,7 +905,34 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
)
|
||||
|
||||
if is_block and transposed:
|
||||
# The oracle gates column-major scales on TRTLLM fp8-block.
|
||||
assert quant_config.is_scale_transposed
|
||||
assert not make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.DEEPGEMM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
).is_scale_transposed
|
||||
# no_dp_ep emits column-major [M, G] scales (physical [G, M]), so
|
||||
# the experts' a1q_scale.t().contiguous() is a no-op.
|
||||
_, a1q_scale, _, _, _ = kernel.prepare_finalize.prepare(
|
||||
a, topk_weights, topk_ids, e, None, False, quant_config
|
||||
)
|
||||
assert a1q_scale.t().is_contiguous()
|
||||
# Bitwise-equal to running with row-major scales.
|
||||
row_major = replace(quant_config, is_scale_transposed=False)
|
||||
torch.testing.assert_close(
|
||||
apply(kernel), apply(make_kernel(row_major)), atol=0, rtol=0
|
||||
)
|
||||
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: apply(kernel),
|
||||
router_weights=topk_weights,
|
||||
)
|
||||
|
||||
@@ -986,3 +1030,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
)
|
||||
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "n", "k"),
|
||||
[
|
||||
(MoEActivation.SILU, 1024, 1024),
|
||||
(MoEActivation.SILU, 928, 1024),
|
||||
(MoEActivation.SILU, 1024, 2688),
|
||||
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
|
||||
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
|
||||
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("m", [1, 32])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_mxfp8_moe_pads_intermediate(
|
||||
m: int, activation: MoEActivation, n: int, k: int, workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
|
||||
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
|
||||
not, are padded at load and, with the activations padded and the output
|
||||
cropped as the MoE runner does, must still match the unquantized reference
|
||||
on the dequantized weights."""
|
||||
e, topk = 8, 2
|
||||
is_gated = activation.is_gated
|
||||
block_shape = [1, 32]
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
|
||||
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
|
||||
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=is_gated, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=activation,
|
||||
)
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
layer,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
padded_n = layer.moe_config.intermediate_size_per_partition
|
||||
padded_k = layer.moe_config.hidden_dim
|
||||
assert padded_n == round_up(n, 128)
|
||||
assert padded_k == round_up(k, 256)
|
||||
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=padded_k,
|
||||
intermediate_size=padded_n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
# The monolithic experts pass the activation type to the kernel; the
|
||||
# modular ones always run it gated.
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=True,
|
||||
),
|
||||
TrtLlmFp8ExpertsMonolithic(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
|
||||
output = kernel.apply_monolithic(
|
||||
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
router_logits=logits,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)[:, :k]
|
||||
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
|
||||
reference = fused_experts(
|
||||
a,
|
||||
w1_ref,
|
||||
w2_ref,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
check_accuracy(ref_output=reference, actual_output=output)
|
||||
|
||||
@@ -42,13 +42,15 @@ ROCM_AVAILABLE = current_platform.is_rocm()
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = False
|
||||
ROCM_AITER_AVAILABLE = is_aiter_found()
|
||||
ROCM_GFX950 = False
|
||||
ROCM_GFX1250 = False
|
||||
|
||||
if ROCM_AVAILABLE:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = has_triton_kernels()
|
||||
ROCM_GFX950 = on_gfx950()
|
||||
ROCM_GFX1250 = on_gfx1250()
|
||||
|
||||
if ROCM_AITER_AVAILABLE:
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
@@ -1568,6 +1570,15 @@ ROCM_BACKEND_CONFIGS = {
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": True,
|
||||
},
|
||||
"AITER_TRITON_MXFP4_MXFP4": {
|
||||
"activation": "SILU",
|
||||
"act_type": "mxfp4",
|
||||
"rtol": 1.0,
|
||||
"percent": 0.8,
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": False,
|
||||
"requires_gfx1250": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1607,6 +1618,8 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
pytest.skip(f"Backend {backend_name} requires AITER")
|
||||
if config["requires_gfx950"] and not ROCM_GFX950:
|
||||
pytest.skip(f"Backend {backend_name} requires GFX950")
|
||||
if config.get("requires_gfx1250") and not ROCM_GFX1250:
|
||||
pytest.skip(f"Backend {backend_name} requires GFX1250")
|
||||
|
||||
import vllm.distributed.parallel_state as ps
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
@@ -219,3 +219,169 @@ def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
|
||||
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
|
||||
)
|
||||
assert torch.allclose(topk_ids_original, topk_ids_compiled)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_custom_op_registration():
|
||||
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
|
||||
|
||||
|
||||
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
|
||||
ids_sorted, order = torch.sort(ids, dim=-1)
|
||||
weights_sorted = torch.gather(weights, 1, order)
|
||||
return weights_sorted, ids_sorted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
|
||||
@pytest.mark.parametrize("num_experts", [256, 512])
|
||||
@pytest.mark.parametrize("topk", [8, 10])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
def test_rocm_aiter_topk_gating_matches_softmax_reference(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
):
|
||||
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
scores = torch.softmax(gating_output.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
|
||||
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
|
||||
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
# BF16 logits can tie at the top-k boundary, so a different tied expert is
|
||||
# valid. Check selection against the cutoff and weights against the scores
|
||||
# of the experts the kernel actually selected.
|
||||
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(topk_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected_scores = torch.gather(scores, 1, topk_ids.long())
|
||||
assert (selected_scores >= cutoff).all()
|
||||
expected_weights = selected_scores
|
||||
if renormalize:
|
||||
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_shared_expert_suffix():
|
||||
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
|
||||
|
||||
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
|
||||
column, weights wide enough for the suffix, and the shared id prefilled.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
|
||||
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
|
||||
shared[0, 0] = 40
|
||||
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
|
||||
width = topk + num_shared
|
||||
shared_ids = torch.arange(
|
||||
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
|
||||
)
|
||||
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
|
||||
topk_ids[:, topk:] = shared_ids
|
||||
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids[:, :topk],
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
True,
|
||||
num_shared,
|
||||
"sigmoid",
|
||||
)
|
||||
|
||||
scores = torch.softmax(routed.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
routed_ids = topk_ids[:, :topk]
|
||||
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(routed_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected = torch.gather(scores, 1, routed_ids.long())
|
||||
assert (selected >= cutoff).all()
|
||||
expected = selected / selected.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(
|
||||
topk_weights[:, topk:],
|
||||
torch.sigmoid(shared.float()),
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
)
|
||||
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
|
||||
token, expert, topk, renormalize = 4, 512, 10, True
|
||||
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
|
||||
device = gating_output.device
|
||||
|
||||
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
|
||||
return torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_topk_gating,
|
||||
(
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
gating_output,
|
||||
),
|
||||
kwargs={
|
||||
"renormalize": renormalize,
|
||||
"num_shared_experts": 0,
|
||||
"shared_expert_scoring_func": "",
|
||||
},
|
||||
test_utils=("test_faketensor"),
|
||||
)
|
||||
|
||||
compiled_fn = torch.compile(
|
||||
gating_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
def alloc():
|
||||
return (
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
w0, i0, t0 = alloc()
|
||||
w1, i1, t1 = alloc()
|
||||
gating_fn(gating_output, w0, i0, t0)
|
||||
compiled_fn(gating_output, w1, i1, t1)
|
||||
w0, i0 = _sort_routing(w0, i0)
|
||||
w1, i1 = _sort_routing(w1, i1)
|
||||
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
|
||||
assert torch.equal(i0, i1)
|
||||
|
||||
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables must reach backend selection; modular AITER remains eligible."""
|
||||
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.platforms import rocm
|
||||
|
||||
# Exercise both native MX and monolithic AITER selection on any ROCm device.
|
||||
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
|
||||
),
|
||||
prefix="hash_routing_selection",
|
||||
)
|
||||
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
|
||||
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
else:
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables reach backend selection and hash-capable kernels."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
layer, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, has_hash_routing, backend
|
||||
)
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
# Monolithic kernels route internally, so they read the table from the layer.
|
||||
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
def test_hash_routing_rejects_monolithic_without_support(
|
||||
dist_init, default_vllm_config, monkeypatch
|
||||
):
|
||||
"""A monolithic kernel that does not implement hash routing is rejected."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
"_supports_hash_routing",
|
||||
staticmethod(lambda: False),
|
||||
)
|
||||
_, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
|
||||
)
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
|
||||
|
||||
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
|
||||
|
||||
@@ -50,7 +50,7 @@ SKINNY_GEMM_PASS_RATES = {
|
||||
}
|
||||
PRESHUFFLED_SHAPES = [
|
||||
(64, 4096, 8192),
|
||||
(32, 8192, 16384),
|
||||
(32, 16384, 16384),
|
||||
]
|
||||
|
||||
|
||||
@@ -421,8 +421,12 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
M, K, N = shape
|
||||
|
||||
assert M <= 64
|
||||
# The predicate takes K as packed bytes, matching `weight.shape[1]`.
|
||||
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
|
||||
# must agree with the config the kernel itself resolves.
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
|
||||
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
|
||||
|
||||
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
@@ -464,6 +468,18 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
_assert_deterministic(run_preshuffled, n_runs=3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
|
||||
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
|
||||
has no tuned preshuffled config even though twice its K does."""
|
||||
_assert_aiter_supported()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
|
||||
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_gemm_a4w4_determinism():
|
||||
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
|
||||
|
||||
@@ -14,6 +14,7 @@ These tests cover:
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -157,6 +158,104 @@ def test_dspark_context_kv_matches_query_insert(
|
||||
torch.testing.assert_close(kv, before, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("kv_mxfp8", [False, True])
|
||||
def test_v41_pcp_swa_cache_matches_non_pcp_writer(kv_mxfp8, monkeypatch):
|
||||
"""PCP SWA writes must preserve the established CUDA cache bytes."""
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
torch.manual_seed(43)
|
||||
block_size = 64
|
||||
num_local_tokens = 5
|
||||
num_tokens = 2 * num_local_tokens
|
||||
row_size = 528 if kv_mxfp8 else 584
|
||||
num_blocks = 2
|
||||
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
|
||||
backing = torch.full(
|
||||
(num_blocks, page_stride), 165, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
cache = backing.as_strided(
|
||||
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
|
||||
)
|
||||
reference_backing = backing.clone()
|
||||
reference = reference_backing.as_strided(cache.shape, cache.stride())
|
||||
|
||||
local_kv = torch.randn(num_local_tokens, 512, dtype=torch.bfloat16, device="cuda")
|
||||
remote_kv = torch.randn_like(local_kv)
|
||||
gathered_kv = torch.cat((local_kv, remote_kv))
|
||||
local_positions = torch.arange(num_local_tokens, device="cuda") + 3
|
||||
cache_positions = torch.cat((local_positions, local_positions + 11))
|
||||
cache_slots = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device="cuda"
|
||||
)[:num_tokens]
|
||||
cache_slots[2] = -1
|
||||
angles = torch.randn(32, 32, device="cuda")
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
num_heads = 8
|
||||
q = torch.randn(
|
||||
num_local_tokens,
|
||||
num_heads,
|
||||
512,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch.zeros(num_tokens, num_heads, 512, dtype=torch.bfloat16, device="cuda"),
|
||||
gathered_kv,
|
||||
reference.view(num_blocks, -1),
|
||||
cache_slots,
|
||||
cache_positions,
|
||||
cos_sin,
|
||||
num_heads,
|
||||
1e-20,
|
||||
block_size,
|
||||
False,
|
||||
kv_mxfp8,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, remote_kv), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = SimpleNamespace(
|
||||
slot_mapping=torch.full(
|
||||
(num_local_tokens,), -1, dtype=torch.int64, device="cuda"
|
||||
),
|
||||
cache_slot_mapping=cache_slots,
|
||||
cache_positions=cache_positions,
|
||||
block_size=block_size,
|
||||
)
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
swa_cache_layer=SimpleNamespace(
|
||||
prefix="swa", kv_cache=cache, block_size=block_size
|
||||
),
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
|
||||
_flashinfer_fp8_kv_scale=None,
|
||||
kv_mxfp8=kv_mxfp8,
|
||||
accepts_unnormed_unroped_query=False,
|
||||
n_local_heads=num_heads,
|
||||
padded_heads=num_heads,
|
||||
eps=1e-20,
|
||||
head_dim=512,
|
||||
)
|
||||
DeepseekV4Attention._fused_qnorm_rope_kv_insert(
|
||||
attention, q, local_kv, local_positions, {"swa": metadata}
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
backing.view(torch.uint8),
|
||||
reference_backing.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="graph capture coverage"
|
||||
)
|
||||
@@ -360,7 +459,7 @@ def _mxfp8_record_reference(
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
|
||||
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
|
||||
in bf16, so a token is [512 fp8][16 UE8M0].
|
||||
|
||||
Byte-exact against the reference quantizer, including the rows and page
|
||||
padding the kernel must leave alone.
|
||||
@@ -398,9 +497,8 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
values, scales = _mxfp8_record_reference(
|
||||
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
|
||||
)
|
||||
expected[page, row * 512 : (row + 1) * 512] = values
|
||||
scale_offset = cache_block * 512 + row * 16
|
||||
expected[page, scale_offset : scale_offset + 16] = scales
|
||||
expected[page, row * 528 : row * 528 + 512] = values
|
||||
expected[page, row * 528 + 512 : (row + 1) * 528] = scales
|
||||
|
||||
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
|
||||
|
||||
@@ -510,14 +608,13 @@ def test_v41_mxfp8_cache_round_trip():
|
||||
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
|
||||
page, row = divmod(t, block_size)
|
||||
torch.testing.assert_close(
|
||||
cache[page, row * 512 : (row + 1) * 512],
|
||||
cache[page, row * 528 : row * 528 + 512],
|
||||
values.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
scale_offset = block_size * 512 + row * 16
|
||||
torch.testing.assert_close(
|
||||
cache[page, scale_offset : scale_offset + 16],
|
||||
cache[page, row * 528 + 512 : (row + 1) * 528],
|
||||
(scales.log2() + 127).to(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
@@ -564,7 +661,8 @@ def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tenso
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales
|
||||
per token.
|
||||
|
||||
Scales are byte-exact against the reference (``amax / 6`` clamped to the
|
||||
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
|
||||
@@ -604,17 +702,13 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
amax = rotated.view(32, 16).abs().amax(-1)
|
||||
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
|
||||
|
||||
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
|
||||
record = cache_backing[page, row * 288 : (row + 1) * 288]
|
||||
got_scales = record[256:]
|
||||
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
|
||||
decoded = _decode_nvfp4_row(
|
||||
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
|
||||
)
|
||||
decoded = _decode_nvfp4_row(record[:256], got_scales)
|
||||
step = scale.float().repeat_interleave(16)
|
||||
assert ((decoded - rotated).abs() <= step).all()
|
||||
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
|
||||
page, row * 256 : (row + 1) * 256
|
||||
]
|
||||
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
|
||||
untouched[page, row * 288 : (row + 1) * 288] = record
|
||||
# Rows the kernel must not have touched, page padding included.
|
||||
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
|
||||
|
||||
@@ -670,10 +764,8 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
expected = _decode_nvfp4_row(
|
||||
backing[:, : block_size * 256].reshape(-1, 256),
|
||||
backing[:, block_size * 256 : block_size * 288].reshape(-1, 32),
|
||||
)
|
||||
records = backing[:, : block_size * 288].reshape(-1, 288)
|
||||
expected = _decode_nvfp4_row(records[:, :256], records[:, 256:])
|
||||
|
||||
def check(shorter_by):
|
||||
assert (storage[:, num_tokens + offset :] == -123).all()
|
||||
@@ -702,6 +794,72 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
|
||||
check(1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The V4.1 records are SM100-only",
|
||||
)
|
||||
@pytest.mark.parametrize("record", ["swa_fp8", "compressed_fp8", "compressed_fp4"])
|
||||
def test_v41_cache_writers_match_flashmla_decode(record: str):
|
||||
"""FlashMLA and vLLM's gathers read back the token a V4.1 writer put mid
|
||||
page in a padded block, so all agree on the interleaved record layout."""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
dequantize_and_gather_k_cache,
|
||||
dequantize_and_gather_k_cache_triton,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
device, block_size, topk = "cuda", 64, 64
|
||||
slot = block_size + 37
|
||||
bytes_per_token = 288 if record == "compressed_fp4" else 528
|
||||
torch.manual_seed(0)
|
||||
kv = torch.randn(1, 512, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.tensor([slot], device=device)
|
||||
# cos = 1, sin = 0: RoPE is the identity, so the cache should hold `kv`.
|
||||
cos_sin = torch.zeros(slot + 1, 64, device=device)
|
||||
cos_sin[:, :32] = 1
|
||||
# Padded like a packed block: a multiple of the record and of 512 B.
|
||||
cache = torch.zeros(2, block_size + 32, bytes_per_token, dtype=torch.uint8)
|
||||
cache = cache.to(device)[:, :block_size]
|
||||
if record == "swa_fp8":
|
||||
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
|
||||
kv, cache, positions, positions, cos_sin, block_size, None, True
|
||||
)
|
||||
else:
|
||||
rope_quant_insert(kv, positions, cos_sin, cache, positions, 1)
|
||||
|
||||
indices = torch.full((1, 1, topk), -1, dtype=torch.int32, device=device)
|
||||
indices[..., 0] = slot
|
||||
extra = None
|
||||
if record == "compressed_fp4":
|
||||
cache, extra = torch.zeros(2, block_size, 528).to(cache), cache
|
||||
out, _ = fm.flash_mla_with_kvcache(
|
||||
q=torch.randn(1, 1, 64, 512, dtype=torch.bfloat16, device=device),
|
||||
k_cache=cache.unsqueeze(2),
|
||||
block_table=None,
|
||||
cache_seqlens=None,
|
||||
head_dim_v=512,
|
||||
tile_scheduler_metadata=fm.get_mla_metadata()[0],
|
||||
is_fp8_kvcache=True,
|
||||
indices=indices if extra is None else torch.full_like(indices, -1),
|
||||
extra_k_cache=None if extra is None else extra.unsqueeze(2),
|
||||
extra_indices_in_kvcache=None if extra is None else indices,
|
||||
)
|
||||
for gather in (dequantize_and_gather_k_cache, dequantize_and_gather_k_cache_triton):
|
||||
rows = torch.zeros(1, slot + 1, 512, dtype=torch.bfloat16, device=device)
|
||||
block_table = torch.arange(2, dtype=torch.int32, device=device)[None]
|
||||
seq_lens = torch.tensor([slot + 1], dtype=torch.int32, device=device)
|
||||
k_cache = cache if extra is None else extra
|
||||
gather(rows, k_cache, seq_lens, None, block_table, block_size, offset=0)
|
||||
torch.testing.assert_close(
|
||||
out[0, 0], rows[0, slot].expand(64, -1), rtol=0, atol=0
|
||||
)
|
||||
cos = torch.nn.functional.cosine_similarity(rows[0, slot].float(), kv[0].float(), 0)
|
||||
assert cos > (0.98 if extra is not None else 0.999)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
@@ -796,6 +954,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
capacity = 8
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 16
|
||||
vllm_config.parallel_config.prefill_context_parallel_size = 1
|
||||
spec = CircularBufferSpec(
|
||||
block_size=capacity,
|
||||
num_kv_heads=1,
|
||||
@@ -832,6 +991,67 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
|
||||
|
||||
|
||||
def test_v41_pcp_compressor_latent_is_gathered_for_cache_writes(monkeypatch):
|
||||
"""PCP compresses locally and publishes the gathered latent to both caches."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
local_latent = torch.arange(8, dtype=torch.float32).view(2, 4)
|
||||
cache_positions = torch.tensor([0, 1, 10, 11])
|
||||
observed = {}
|
||||
|
||||
class Compressor:
|
||||
def __call__(self, kv_score, positions):
|
||||
return local_latent
|
||||
|
||||
def insert_cache(self, latent, positions, rotary_emb):
|
||||
observed["insert"] = (latent, positions)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
observed["indexer"] = (latent, kwargs)
|
||||
return None, None, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, tensor + 10), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = {"swa": SimpleNamespace(cache_positions=cache_positions)}
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
compressor=Compressor(),
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=None,
|
||||
ln_events=[None, None],
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa"),
|
||||
rotary_emb=SimpleNamespace(),
|
||||
indexer_rotary_emb=SimpleNamespace(),
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
_wq_b_proj=lambda qr, scale: qr,
|
||||
_fused_qnorm_rope_kv_insert=lambda q, kv, pos, meta: q,
|
||||
_sparse_indexer_and_attn=lambda *args: None,
|
||||
)
|
||||
rows = torch.zeros(2, 4)
|
||||
positions = torch.arange(2)
|
||||
with override_forward_context(ForwardContext({}, metadata, {})):
|
||||
DeepseekV4Attention._prepare_and_attn(
|
||||
attention, rows, rows, rows, None, rows, rows, positions, rows
|
||||
)
|
||||
|
||||
expected_global = torch.cat((local_latent, local_latent + 10))
|
||||
insert_latent, insert_positions = observed["insert"]
|
||||
torch.testing.assert_close(insert_latent, expected_global)
|
||||
torch.testing.assert_close(insert_positions, cache_positions)
|
||||
indexer_latent, indexer_kwargs = observed["indexer"]
|
||||
torch.testing.assert_close(indexer_latent, local_latent)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_latent"], expected_global)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_positions"], cache_positions)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
|
||||
@pytest.mark.parametrize(
|
||||
"use_aux,use_graph", [(False, False), (True, False), (True, True)]
|
||||
@@ -871,6 +1091,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
context = ForwardContext({}, metadata, {})
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.pcp_size = 1
|
||||
compressor.head_dim, compressor.rope_head_dim, compressor.compress_ratio = (
|
||||
512,
|
||||
64,
|
||||
@@ -896,7 +1117,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
use_fp4_kv=False,
|
||||
)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale):
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
DeepseekV4Indexer._produce_k(indexer, latent, positions, rotary)
|
||||
return None, None, None
|
||||
|
||||
@@ -906,6 +1127,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
|
||||
auxiliary = [torch.cuda.Stream()] if use_aux else None
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=False,
|
||||
compressor=compressor,
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=auxiliary,
|
||||
@@ -963,6 +1185,291 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
previous = expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
@pytest.mark.parametrize(
|
||||
"lengths,prefilling,starts",
|
||||
[
|
||||
([9, 1], [True, False], [1, 4]),
|
||||
([1, 1], [False, False], [16, 3]),
|
||||
([3, 1], [True, False], [1, 0]),
|
||||
],
|
||||
)
|
||||
def test_v41_pcp_ring_links_match_request_positions(
|
||||
pcp_size, lengths, prefilling, starts, monkeypatch
|
||||
):
|
||||
"""Cross-chunk predecessors and open ring writes must belong to the request."""
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import (
|
||||
CompressorMetadata,
|
||||
CompressorMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
monkeypatch.setattr(
|
||||
pcp_manager_module,
|
||||
"async_tensor_h2d",
|
||||
lambda array, **kwargs: torch.from_numpy(array),
|
||||
)
|
||||
manager = PCPManager(pcp_size, 0, torch.device("cpu"), shard_decode_requests=True)
|
||||
query_start = np.array([0, *np.cumsum(lengths)], dtype=np.int32)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
np.array(lengths), np.array(starts), np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
capacity = 8
|
||||
blocks = [1, (1 << 20) + 2]
|
||||
num_rows = min(width, 2 * len(lengths))
|
||||
metadata_by_rank = []
|
||||
positions_by_rank = []
|
||||
gathered_keys = []
|
||||
for local_segments in segments:
|
||||
positions = torch.zeros(width, dtype=torch.int64)
|
||||
slots = torch.full((width,), -1, dtype=torch.int64)
|
||||
local_query_start = [0]
|
||||
keys = [-1] * num_rows
|
||||
for row, segment in enumerate(local_segments):
|
||||
req = segment.global_batch_req_idx
|
||||
begin = starts[req] + segment.global_batch_slice.start - query_start[req]
|
||||
local = segment.rank_local_batch_slice
|
||||
positions[local] = torch.arange(begin, begin + segment.num_tokens)
|
||||
slots[local] = blocks[req] * capacity + positions[local] % capacity
|
||||
local_query_start.append(local.stop)
|
||||
keys[row] = (blocks[req] << 32) | int(positions[local.stop - 1])
|
||||
if not local_segments:
|
||||
local_query_start.append(0)
|
||||
metadata_by_rank.append(
|
||||
CompressorMetadata(
|
||||
slot_mapping=slots,
|
||||
query_start_loc=torch.tensor(local_query_start, dtype=torch.int32),
|
||||
token_to_req_indices=torch.empty(0, dtype=torch.int32),
|
||||
)
|
||||
)
|
||||
positions_by_rank.append(positions)
|
||||
gathered_keys.extend(keys)
|
||||
|
||||
keys = torch.tensor(gathered_keys, dtype=torch.int64)
|
||||
contributed_keys = []
|
||||
|
||||
def all_gather(tensor, dim):
|
||||
contributed_keys.append(tensor.clone())
|
||||
return keys
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
builder = SimpleNamespace(
|
||||
capacity=capacity,
|
||||
device=torch.device("cpu"),
|
||||
vllm_config=SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=len(lengths))
|
||||
),
|
||||
)
|
||||
expected_rows = {
|
||||
block: max(key for key in gathered_keys if key >> 32 == block)
|
||||
for block in blocks
|
||||
}
|
||||
expected_slots = [
|
||||
block * capacity + (key & 0xFFFFFFFF) % capacity
|
||||
for block, key in expected_rows.items()
|
||||
if key % 2 == 0
|
||||
]
|
||||
for metadata, positions in zip(metadata_by_rank, positions_by_rank):
|
||||
num_reqs = metadata.query_start_loc.numel() - 1
|
||||
CompressorMetadataBuilder._link_pcp_chunks(
|
||||
builder, metadata, num_reqs, positions
|
||||
)
|
||||
assert metadata.prev_row_indices is not None
|
||||
assert metadata.gather_tokens is not None
|
||||
assert metadata.ring_write_slots is not None
|
||||
for row, token in enumerate(metadata.gather_tokens.tolist()):
|
||||
key = int(contributed_keys[-1][row])
|
||||
if key >= 0:
|
||||
assert int(metadata.slot_mapping[token]) // capacity == key >> 32
|
||||
assert int(positions[token]) == key & 0xFFFFFFFF
|
||||
for start, prev in zip(
|
||||
metadata.query_start_loc[:-1].tolist(), metadata.prev_row_indices.tolist()
|
||||
):
|
||||
position = int(positions[start])
|
||||
block = int(metadata.slot_mapping[start]) // capacity
|
||||
wanted = (block << 32) | (position - 1)
|
||||
if position % 2 == 1 and wanted in gathered_keys:
|
||||
assert prev >= 0 and gathered_keys[prev] == wanted
|
||||
else:
|
||||
assert prev == -1
|
||||
writes = metadata.ring_write_slots.tolist()
|
||||
assert sorted(slot for slot in writes if slot >= 0) == sorted(expected_slots)
|
||||
for row, slot in enumerate(writes):
|
||||
if slot >= 0:
|
||||
assert gathered_keys[row] == expected_rows[slot // capacity]
|
||||
assert (gathered_keys[row] & 0xFFFFFFFF) % capacity == slot % capacity
|
||||
torch.testing.assert_close(torch.cat(contributed_keys), keys)
|
||||
|
||||
|
||||
def _build_pcp_ring_metadata(ranks, all_gather, spec, config, device, monkeypatch):
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
for local in ranks:
|
||||
builder = CompressorMetadataBuilder(spec, ["ring"], config, device)
|
||||
local.metadata = builder.build(0, local.common)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
def test_v41_pcp_compressor_preserves_pairs_and_decode_state(pcp_size, monkeypatch):
|
||||
"""Pairs split across PCP chunks and decode owner changes keep exact latents."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
fused_save_compress_norm,
|
||||
)
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
norm = torch.randn(512, dtype=torch.bfloat16, device=device)
|
||||
# Block 0 is reserved as the null block; requests use physical blocks 1/2.
|
||||
reference_state = torch.zeros(3, 8, 1024, device=device)
|
||||
rank_states = [reference_state.clone() for _ in range(pcp_size)]
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=pcp_size),
|
||||
speculative_config=None,
|
||||
)
|
||||
spec = CircularBufferSpec(
|
||||
block_size=8, num_kv_heads=1, head_size=1024, head_size_v=0, dtype=torch.float32
|
||||
)
|
||||
starts = np.zeros(2, dtype=np.int32)
|
||||
for lengths, prefilling in [
|
||||
([7, 3], [True, True]),
|
||||
([9, 1], [True, False]),
|
||||
([1, 1], [False, False]),
|
||||
([1, 1], [False, False]),
|
||||
]:
|
||||
lengths = np.array(lengths, dtype=np.int32)
|
||||
query_start = np.concatenate(([0], np.cumsum(lengths))).astype(np.int32)
|
||||
qsl = torch.from_numpy(query_start).to(device)
|
||||
positions = torch.cat(
|
||||
[torch.arange(s, s + n, device=device) for s, n in zip(starts, lengths)]
|
||||
)
|
||||
req_ids = torch.repeat_interleave(
|
||||
torch.arange(2, device=device, dtype=torch.int32),
|
||||
torch.from_numpy(lengths).to(device),
|
||||
)
|
||||
slots = (req_ids.long() + 1) * 8 + positions % 8
|
||||
raw = torch.randn(sum(lengths), 1024, device=device)
|
||||
expected = torch.empty(sum(lengths), 512, dtype=torch.bfloat16, device=device)
|
||||
fused_save_compress_norm(
|
||||
raw,
|
||||
positions,
|
||||
reference_state,
|
||||
slots,
|
||||
qsl,
|
||||
req_ids,
|
||||
norm,
|
||||
1e-20,
|
||||
2,
|
||||
expected,
|
||||
)
|
||||
|
||||
ranks = []
|
||||
for rank in range(pcp_size):
|
||||
manager = PCPManager(pcp_size, rank, device, shard_decode_requests=True)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
lengths, starts, np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
local_indices = manager._padded_gather_idx[
|
||||
rank * width : (rank + 1) * width
|
||||
]
|
||||
local_lengths = [s.num_tokens for s in segments[rank]] or [0]
|
||||
local_qsl = torch.tensor([0, *np.cumsum(local_lengths)], dtype=torch.int32)
|
||||
local_reqs = [s.global_batch_req_idx for s in segments[rank]] or [0]
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=local_qsl.to(device),
|
||||
query_start_loc_cpu=local_qsl,
|
||||
seq_lens=torch.zeros(len(local_reqs), dtype=torch.int32, device=device),
|
||||
num_reqs=len(local_reqs),
|
||||
num_actual_tokens=counts[rank],
|
||||
max_query_len=max(local_lengths),
|
||||
max_seq_len=int(max(starts + lengths)),
|
||||
block_table_tensor=torch.tensor(
|
||||
np.array(local_reqs) + 1, dtype=torch.int32, device=device
|
||||
).unsqueeze(1),
|
||||
slot_mapping=torch.full((width * pcp_size,), -1, device=device),
|
||||
positions=positions[local_indices],
|
||||
)
|
||||
ranks.append(
|
||||
SimpleNamespace(
|
||||
indices=local_indices[: counts[rank]],
|
||||
raw=raw[local_indices],
|
||||
positions=common.positions,
|
||||
common=common,
|
||||
)
|
||||
)
|
||||
|
||||
# Each rank's builder contributes its last-row candidates to a gather.
|
||||
candidates: list[torch.Tensor] = []
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: out.append(t.clone()) or t.repeat(pcp_size),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: torch.cat(out),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
gathered_rows = torch.cat([r.raw[r.metadata.gather_tokens] for r in ranks])
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda rows=gathered_rows: SimpleNamespace(all_gather=lambda t, dim: rows),
|
||||
)
|
||||
for rank, local in enumerate(ranks):
|
||||
compressor = SimpleNamespace(
|
||||
pcp_size=pcp_size,
|
||||
head_dim=512,
|
||||
compress_ratio=2,
|
||||
rms_norm_eps=1e-20,
|
||||
norm=SimpleNamespace(weight=norm),
|
||||
state_cache=SimpleNamespace(prefix="ring", kv_cache=rank_states[rank]),
|
||||
)
|
||||
context = ForwardContext({}, {"ring": local.metadata}, {})
|
||||
with override_forward_context(context):
|
||||
actual = DeepseekCompressor.forward(
|
||||
compressor, local.raw, local.positions
|
||||
)
|
||||
assert isinstance(actual, torch.Tensor)
|
||||
valid = (positions[local.indices] + 1) % 2 == 0
|
||||
torch.testing.assert_close(
|
||||
actual[: local.indices.numel()][valid],
|
||||
expected[local.indices][valid],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
for state in rank_states[1:]:
|
||||
torch.testing.assert_close(state, rank_states[0], rtol=0, atol=0)
|
||||
starts += lengths
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
@@ -502,11 +502,12 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
def test_v2_model_state_gathers_lookback_window():
|
||||
@pytest.mark.parametrize("pcp", [False, True])
|
||||
def test_v2_model_state_gathers_lookback_window(pcp):
|
||||
"""The window is gathered on device from the runner's token history."""
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
|
||||
depth, max_num_reqs, max_model_len = 3, 4, 16
|
||||
depth, max_num_reqs, max_model_len = 3, 6, 16
|
||||
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
|
||||
state.rope_state = None
|
||||
state.supports_mm_inputs = False
|
||||
@@ -527,15 +528,23 @@ def test_v2_model_state_gathers_lookback_window():
|
||||
# Batch rows -> request state rows: a request two decode steps in, a fresh
|
||||
# request, and one at the end of a 5-token prompt.
|
||||
input_batch = SimpleNamespace(
|
||||
idx_mapping=torch.tensor([2, 0, 3], dtype=torch.int64, device="cuda")
|
||||
idx_mapping=torch.tensor(
|
||||
[2, 0, 3, 2] if pcp else [2, 0, 3], dtype=torch.int64, device="cuda"
|
||||
),
|
||||
query_start_loc=torch.arange(5 if pcp else 4, dtype=torch.int32, device="cuda"),
|
||||
positions=torch.tensor([8, 0, 6, 5] if pcp else [8, 0, 6], device="cuda"),
|
||||
)
|
||||
window = state.prepare_inputs(input_batch, req_states)["lookback_token_ids"]
|
||||
assert window.cpu().tolist() == [
|
||||
expected = [
|
||||
[22, 21, 16],
|
||||
[-1, -1, -1],
|
||||
[0, 0, 0],
|
||||
[-1, -1, -1],
|
||||
]
|
||||
if pcp:
|
||||
# The second PCP chunk of request 2 starts at 5, not at its global 8.
|
||||
expected.append([15, 14, 13])
|
||||
expected.extend([[-1] * depth] * (max_num_reqs - len(expected)))
|
||||
assert window.cpu().tolist() == expected
|
||||
# Graph capture runs on the dummy inputs and replays read the same buffer.
|
||||
dummy = state.prepare_dummy_inputs(num_reqs=3, num_tokens=3)["lookback_token_ids"]
|
||||
assert dummy is window and torch.all(dummy == -1)
|
||||
|
||||
@@ -525,9 +525,8 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
|
||||
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
|
||||
assert ((rec_fused - rec_ref).abs() <= step).all()
|
||||
# The NoPE dims see no rotation at all, so they must be byte-identical.
|
||||
nope_bytes = slice(0, block_size * HEAD_DIM)
|
||||
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
ref_rows = k_cache_ref.view(num_blocks, block_size, V41_HEAD_BYTES)
|
||||
fused_rows = k_cache_fused.view(num_blocks, block_size, V41_HEAD_BYTES)
|
||||
torch.testing.assert_close(
|
||||
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
|
||||
)
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size"),
|
||||
[(3072, 256), (4096, 8), (4096, 192), (6144, 128), (6144, 256)],
|
||||
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
|
||||
)
|
||||
def test_rocm_gfx950_enables_fp32_router_gemm(
|
||||
monkeypatch, input_size: int, output_size: int
|
||||
@@ -147,7 +147,6 @@ def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
|
||||
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
|
||||
[
|
||||
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
|
||||
pytest.param(2048, 64, torch.float32, False, True, id="shape"),
|
||||
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
|
||||
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
|
||||
],
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
rocm_fp32_router_gemm as rocm_router,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import (
|
||||
fp32_router_gemm_dispatch_impl,
|
||||
)
|
||||
@@ -19,6 +22,11 @@ SHAPES = [
|
||||
(4096, 192),
|
||||
(6144, 128),
|
||||
(6144, 256),
|
||||
(2560, 512),
|
||||
(2880, 128),
|
||||
(5120, 16),
|
||||
(7168, 384),
|
||||
(1000, 100),
|
||||
]
|
||||
MAX_TOKENS = 32
|
||||
ATOL = 5e-4
|
||||
@@ -80,7 +88,7 @@ def test_rocm_fp32_router_gemm_preserves_topk(
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
|
||||
|
||||
top_k = 2 if (hidden_size, num_experts) == (4096, 8) else 8
|
||||
top_k = 2 if num_experts <= 16 else 8
|
||||
actual = rocm_fp32_router_gemm(x, weight).topk(top_k, dim=-1).indices
|
||||
reference = _reference(x, weight)
|
||||
reference_values, expected = reference.topk(top_k, dim=-1)
|
||||
@@ -113,15 +121,22 @@ def test_rocm_fp32_router_gemm_rejects_invalid_inputs() -> None:
|
||||
weight,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="shape"):
|
||||
rocm_fp32_router_gemm(x[:, :2048].contiguous(), weight[:, :2048].contiguous())
|
||||
with pytest.raises(ValueError, match="hidden sizes"):
|
||||
rocm_fp32_router_gemm(x, weight[:, :2048].contiguous())
|
||||
|
||||
with pytest.raises(ValueError, match="num_experts"):
|
||||
big_weight = torch.empty(1024, 7168, dtype=torch.float32, device=device)
|
||||
rocm_fp32_router_gemm(
|
||||
torch.empty(MAX_TOKENS, 7168, dtype=torch.bfloat16, device=device),
|
||||
big_weight,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="contiguous"):
|
||||
wide_x = torch.randn(4, hidden_size * 2, dtype=torch.bfloat16, device=device)
|
||||
rocm_fp32_router_gemm(wide_x[:, ::2], weight)
|
||||
|
||||
with pytest.raises(ValueError, match="float32"):
|
||||
rocm_fp32_router_gemm(x, weight.to(torch.bfloat16))
|
||||
with pytest.raises(ValueError, match="float32 or bfloat16"):
|
||||
rocm_fp32_router_gemm(x, weight.to(torch.float16))
|
||||
|
||||
with pytest.raises(ValueError, match="bfloat16 or float32"):
|
||||
rocm_fp32_router_gemm(x.to(torch.float16), weight)
|
||||
@@ -226,3 +241,154 @@ def test_rocm_fp32_router_gemm_dynamic_compile_dispatch() -> None:
|
||||
)
|
||||
output = compiled_dispatch(x)
|
||||
torch.testing.assert_close(output, _reference(x, weight), atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", [64, 100, 513, 1000, 2049, 3000])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 5, 32])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_any_hidden_size(num_tokens: int, hidden_size: int):
|
||||
"""Hidden sizes that do not divide the K tile exercise the masked tail."""
|
||||
torch.manual_seed(6100 + num_tokens + hidden_size)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(64, hidden_size, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(
|
||||
rocm_fp32_router_gemm(x, weight), _reference(x, weight), atol=ATOL, rtol=RTOL
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["large", "near_zero", "zero_input", "ones"])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_special_values(case: str) -> None:
|
||||
torch.manual_seed(7100)
|
||||
x = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
if case in ("large", "near_zero"):
|
||||
scale = 100.0 if case == "large" else 1e-4
|
||||
x = (x * scale).to(torch.bfloat16)
|
||||
weight *= scale
|
||||
elif case == "zero_input":
|
||||
x.zero_()
|
||||
else:
|
||||
x.fill_(1.0)
|
||||
output = rocm_fp32_router_gemm(x, weight)
|
||||
expected = _reference(x, weight)
|
||||
assert torch.isfinite(output).all()
|
||||
if case == "zero_input":
|
||||
assert torch.equal(output, torch.zeros_like(output))
|
||||
else:
|
||||
assert ((output - expected).norm() / expected.norm()).item() < 1e-5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_deterministic(num_tokens: int) -> None:
|
||||
torch.manual_seed(8100 + num_tokens)
|
||||
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(384, 7168, dtype=torch.float32, device="cuda")
|
||||
first = rocm_fp32_router_gemm(x, weight)
|
||||
assert torch.equal(first, rocm_fp32_router_gemm(x, weight))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_bf16_weight(
|
||||
num_tokens: int, hidden_size: int, num_experts: int
|
||||
) -> None:
|
||||
torch.manual_seed(9100 + num_tokens + hidden_size + num_experts)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(num_experts, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
# Past this size hipBLASLt beats the kernel for BF16 weights on MI355.
|
||||
eligible = num_tokens * num_experts <= 8192
|
||||
assert rocm_router.can_use_rocm_fp32_router_gemm(x, weight) == eligible
|
||||
if eligible:
|
||||
torch.testing.assert_close(
|
||||
rocm_fp32_router_gemm(x, weight),
|
||||
_reference(x.float(), weight.float()),
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
|
||||
|
||||
def _make_gate_linear(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype=torch.float32,
|
||||
):
|
||||
for module in ("layers.linear", "parameter"):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.{module}.get_tensor_model_parallel_rank", lambda: 0
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.{module}.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
gate = GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
params_dtype=params_dtype,
|
||||
).cuda()
|
||||
with torch.no_grad():
|
||||
gate.weight.normal_()
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
return gate
|
||||
|
||||
|
||||
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32, 33])
|
||||
@torch.inference_mode()
|
||||
def test_gate_linear_uses_rocm_fp32_router_gemm(
|
||||
monkeypatch: pytest.MonkeyPatch, num_tokens: int, x_dtype: torch.dtype
|
||||
) -> None:
|
||||
"""A router shape outside the old tuned list takes the fast path for M<=32."""
|
||||
gate = _make_gate_linear(monkeypatch, 2560, 512)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
calls = []
|
||||
kernel = rocm_router.rocm_fp32_router_gemm
|
||||
|
||||
def spy(hidden_states, router_weight):
|
||||
calls.append(hidden_states.shape[0])
|
||||
return kernel(hidden_states, router_weight)
|
||||
|
||||
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
|
||||
x = torch.randn(num_tokens, 2560, dtype=x_dtype, device="cuda")
|
||||
output, bias = gate(x)
|
||||
|
||||
assert bias is None
|
||||
assert calls == ([num_tokens] if num_tokens <= MAX_TOKENS else [])
|
||||
torch.testing.assert_close(output, _reference(x, gate.weight), atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_experts", "uses_kernel"),
|
||||
[(1, 256, True), (32, 256, True), (33, 256, False), (32, 512, False)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_gate_linear_bf16_weight_uses_rocm_router_gemm(
|
||||
monkeypatch: pytest.MonkeyPatch, num_tokens, num_experts, uses_kernel
|
||||
) -> None:
|
||||
"""BF16-weight gates with FP32 logits use the kernel only where it beats
|
||||
hipBLASLt; everything else keeps the torch.mm path."""
|
||||
gate = _make_gate_linear(monkeypatch, 7168, num_experts, torch.bfloat16)
|
||||
assert gate.allow_cublas_router_gemm
|
||||
calls = []
|
||||
kernel = rocm_router.rocm_fp32_router_gemm
|
||||
|
||||
def spy(hidden_states, router_weight):
|
||||
calls.append(hidden_states.shape[0])
|
||||
return kernel(hidden_states, router_weight)
|
||||
|
||||
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
|
||||
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
output, _ = gate(x)
|
||||
|
||||
assert output.dtype == torch.float32
|
||||
assert calls == ([num_tokens] if uses_kernel else [])
|
||||
torch.testing.assert_close(
|
||||
output, _reference(x.float(), gate.weight.float()), atol=ATOL, rtol=RTOL
|
||||
)
|
||||
|
||||
@@ -830,15 +830,19 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
_assert_same_selection(dense, out, ref)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
def _build_fp4_paged_cache(
|
||||
req_lens: torch.Tensor, page_kv: int, head_dim: int, page_pad: int = 0
|
||||
):
|
||||
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
|
||||
fp4 data first, then their packed UE8M0 scales."""
|
||||
fp4 data first, then their packed UE8M0 scales; ``page_pad`` bytes pad
|
||||
each page like the other layers' pages in a packed block."""
|
||||
device = req_lens.device
|
||||
pages_per = (req_lens + page_kv - 1) // page_kv
|
||||
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
|
||||
head_bytes = head_dim // 2 + 4
|
||||
kv_cache = torch.zeros(
|
||||
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
|
||||
)
|
||||
num_pages, page_kv * head_bytes + page_pad, dtype=torch.uint8, device=device
|
||||
)[:, : page_kv * head_bytes].view(num_pages, page_kv, 1, head_bytes)
|
||||
block_table = torch.zeros(
|
||||
len(req_lens), max_pages, dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -864,7 +868,10 @@ def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
|
||||
@pytest.mark.parametrize("next_n", [1, 2])
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
|
||||
@pytest.mark.parametrize("page_pad", [0, 256])
|
||||
def test_sparse_mqa_logits_paged_matches_dense_masked(
|
||||
next_n: int, capture: bool, page_pad: int
|
||||
):
|
||||
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
|
||||
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
|
||||
one query row each), CUDA graph capture/replay and schedule reuse by a
|
||||
@@ -886,8 +893,10 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
|
||||
.clamp(min=0)
|
||||
.to(torch.int32)
|
||||
)
|
||||
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
|
||||
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
|
||||
kv_cache, block_table = _build_fp4_paged_cache(
|
||||
req_lens, page_kv, head_dim, page_pad
|
||||
)
|
||||
assert kv_cache.stride(0) % 512 == page_pad # padded pages are not 512B
|
||||
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
@@ -17,6 +17,15 @@ if current_platform.is_cuda():
|
||||
from vllm.model_executor.layers import utils
|
||||
|
||||
|
||||
def _on_gfx11() -> bool:
|
||||
"""gfx11 check that is safe to evaluate at collection time on any platform."""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
|
||||
return on_gfx11()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
|
||||
x = torch.randn(1, 64, dtype=torch.float16)
|
||||
@@ -145,6 +154,101 @@ def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
|
||||
"""A weight whose row stride is offset by ``pad`` elements, as produced by
|
||||
``maybe_pad_weight_avoid_cache_aliasing``."""
|
||||
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (weight.shape[1] + pad, 1)
|
||||
return padded
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
|
||||
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
|
||||
# still reach it instead of falling back to torch.nn.functional.linear.
|
||||
x = torch.randn(4, 1024, dtype=torch.float16)
|
||||
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"k,element_size,expected",
|
||||
[
|
||||
(4096, 2, 64), # 8192 B row stride, aliases
|
||||
(12288, 2, 64), # 24576 B row stride, aliases
|
||||
(1024, 2, 64), # smallest aliasing bf16/fp16 K
|
||||
(2560, 2, 0), # 5120 B, no aliasing
|
||||
(11008, 2, 0), # 22016 B, no aliasing
|
||||
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
|
||||
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
|
||||
],
|
||||
)
|
||||
def test_cache_aliasing_pad_elems(k, element_size, expected):
|
||||
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
|
||||
# Padding is gfx11-only, even for a K whose row stride would alias there.
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
|
||||
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (4096 + 64, 1)
|
||||
assert torch.equal(padded, weight)
|
||||
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
|
||||
# and preserves the pointer captured by any graph.
|
||||
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
|
||||
assert repadded.data_ptr() == padded.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight",
|
||||
[
|
||||
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
|
||||
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
|
||||
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
|
||||
],
|
||||
)
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
assert (
|
||||
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
|
||||
== weight.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
|
||||
@@ -164,6 +268,26 @@ def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch,
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
|
||||
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
|
||||
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
|
||||
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
original_wvsplitk = utils.ops.wvSplitK
|
||||
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
|
||||
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
|
||||
|
||||
@@ -123,19 +123,12 @@ def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
|
||||
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
|
||||
dg = SimpleNamespace(
|
||||
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
|
||||
get_block_m_for_mega_moe=lambda *_: 16,
|
||||
)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
|
||||
)
|
||||
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
|
||||
module = SimpleNamespace(
|
||||
gate=SimpleNamespace(input_size=4096, output_size=128),
|
||||
experts=SimpleNamespace(
|
||||
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
|
||||
),
|
||||
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
|
||||
use_sequence_parallel=False,
|
||||
)
|
||||
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
|
||||
|
||||
@@ -139,6 +139,122 @@ def test_topk_softmax_dispatch(use_rocm_aiter: bool):
|
||||
assert topk_func == vllm_topk_softmax
|
||||
|
||||
|
||||
def _topk_gating_launch(
|
||||
num_tokens: int = 4,
|
||||
num_experts: int = 512,
|
||||
topk: int = 10,
|
||||
num_shared_experts: int = 0,
|
||||
weights_width: int | None = None,
|
||||
ids_width: int | None = None,
|
||||
contiguous: bool = True,
|
||||
) -> dict:
|
||||
"""Tensors for one softmax top-k launch, as dispatch_topk_softmax_func sees them."""
|
||||
width = topk + num_shared_experts
|
||||
weights = torch.empty(num_tokens, weights_width or width)
|
||||
ids = torch.empty(num_tokens, ids_width or width, dtype=torch.int32)
|
||||
ids = ids[:, :topk]
|
||||
if contiguous:
|
||||
gating = torch.empty(num_tokens, num_experts + num_shared_experts)
|
||||
else:
|
||||
gating = torch.empty(num_experts + num_shared_experts + 3, num_tokens).t()
|
||||
return dict(topk_weights=weights, topk_indices=ids, gating_output=gating)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"launch, num_shared_experts, scoring_func, expect_gating",
|
||||
[
|
||||
pytest.param({}, 0, "", True, id="plain-softmax"),
|
||||
pytest.param({"num_tokens": 4096}, 0, "", True, id="max-tokens"),
|
||||
pytest.param({"num_tokens": 4097}, 0, "", False, id="too-many-tokens"),
|
||||
pytest.param({"contiguous": False}, 0, "", False, id="non-contiguous"),
|
||||
pytest.param({}, 0, "sigmoid", False, id="scoring-without-shared"),
|
||||
pytest.param({"num_shared_experts": 1}, 1, "sigmoid", True, id="one-shared"),
|
||||
pytest.param({"num_shared_experts": 8}, 8, "sigmoid", True, id="eight-shared"),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 3}, 3, "sigmoid", False, id="unsupported-count"
|
||||
),
|
||||
pytest.param({"num_shared_experts": 1}, 1, "", False, id="no-shared-scoring"),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1}, 1, "softmax", False, id="wrong-shared-scoring"
|
||||
),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1, "weights_width": 10},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="weights-too-narrow",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1, "ids_width": 12},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="row-stride-mismatch",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_tokens": 4097, "num_shared_experts": 1},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="shared-too-many-tokens",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("gating_enabled", [True, False])
|
||||
def test_topk_softmax_dispatch_aiter_topk_gating(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
launch: dict,
|
||||
num_shared_experts: int,
|
||||
scoring_func: str,
|
||||
expect_gating: bool,
|
||||
gating_enabled: bool,
|
||||
):
|
||||
"""AITER launches use topk_gating only where it is enabled, and only when it
|
||||
supports them.
|
||||
|
||||
Every other AITER launch must reach the legacy topk_softmax, so this is the
|
||||
one place the choice is made.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_topk_gating_enabled", lambda: gating_enabled
|
||||
)
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
True,
|
||||
**_topk_gating_launch(**launch),
|
||||
num_shared_experts=num_shared_experts,
|
||||
shared_expert_scoring_func=scoring_func,
|
||||
)
|
||||
if expect_gating and gating_enabled:
|
||||
assert topk_func == rocm_aiter_ops.topk_gating
|
||||
else:
|
||||
assert topk_func == rocm_aiter_ops.topk_softmax
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cdna, fused_moe_enabled, expected",
|
||||
[
|
||||
pytest.param(3, True, True, id="cdna3"),
|
||||
pytest.param(4, True, True, id="cdna4"),
|
||||
pytest.param(2, True, False, id="cdna2"),
|
||||
pytest.param(3, False, False, id="cdna3-aiter-moe-off"),
|
||||
pytest.param(4, False, False, id="cdna4-aiter-moe-off"),
|
||||
],
|
||||
)
|
||||
def test_topk_gating_enabled_by_cdna(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
cdna: int,
|
||||
fused_moe_enabled: bool,
|
||||
expected: bool,
|
||||
):
|
||||
"""topk_gating is on for CDNA3 and CDNA4 with AITER MoE enabled, nowhere else."""
|
||||
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_fused_moe_enabled", lambda: fused_moe_enabled
|
||||
)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.get_cdna_version", lambda: cdna)
|
||||
assert bool(rocm_aiter_ops.is_topk_gating_enabled()) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
|
||||
@@ -7,9 +7,16 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
DPMetadata,
|
||||
ForwardContext,
|
||||
get_forward_context,
|
||||
set_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
@@ -36,6 +43,12 @@ pytestmark = pytest.mark.cpu_test
|
||||
_REC_MODULE = "vllm.model_executor.layers.fused_moe.routed_experts_capturer"
|
||||
|
||||
|
||||
def _forward_context(**kwargs):
|
||||
return ForwardContext(
|
||||
no_compile_layers={}, attn_metadata={}, slot_mapping={}, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def _capturer_with_buffer(
|
||||
*,
|
||||
max_tokens: int = 8,
|
||||
@@ -319,13 +332,120 @@ def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
"""dp_metadata is None: capture writes the full topk_ids rows."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
topk = torch.tensor([[1, 2], [3, 4], [5, 6]], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(dp_metadata=None)
|
||||
ctx = _forward_context()
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
|
||||
assert capturer.device_buffer[3, 0, 0].item() == -1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, num_tokens", [(2, 6), (2, 3), (2, 1), (4, 2)])
|
||||
@pytest.mark.parametrize(
|
||||
"cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL, CUDAGraphMode.PIECEWISE]
|
||||
)
|
||||
def test_single_dp_ep_capture_restores_tp_shards(tp_size, num_tokens, cg_mode):
|
||||
"""Capture all current rows after model-level SP, including buffer reuse."""
|
||||
capturer = _capturer_with_buffer(tp_size=tp_size)
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
tensor_parallel_size=tp_size,
|
||||
enable_expert_parallel=True,
|
||||
use_sequence_parallel_moe=False,
|
||||
is_moe_model=True,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
fast_moe_cold_start=False, static_forward_context={}
|
||||
),
|
||||
)
|
||||
tp_group = Mock()
|
||||
shard_n = (num_tokens + tp_size - 1) // tp_size
|
||||
with (
|
||||
set_forward_context(
|
||||
None,
|
||||
config,
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=cg_mode,
|
||||
),
|
||||
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
|
||||
):
|
||||
assert get_forward_context().dp_metadata is None
|
||||
for offset in (20, 1):
|
||||
expected = torch.arange(num_tokens * 2, dtype=torch.int32).reshape(-1, 2)
|
||||
expected += offset
|
||||
padded = torch.full((shard_n * tp_size, 2), 99, dtype=torch.int32)
|
||||
padded[:num_tokens] = expected
|
||||
tp_group.all_gather.return_value = padded
|
||||
capturer.capture(0, padded[:shard_n])
|
||||
assert torch.equal(
|
||||
capturer.snapshot_routing_data(num_tokens)[:, 0].int(), expected
|
||||
)
|
||||
assert (capturer.device_buffer[num_tokens:, 0] == -1).all()
|
||||
if num_tokens == 1:
|
||||
tp_group.all_gather.assert_not_called()
|
||||
else:
|
||||
assert tp_group.all_gather.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens, num_rows", [(6, 6), (3, 4), (0, 0)])
|
||||
def test_single_dp_tp_capture_does_not_regather_full_batch(num_tokens, num_rows):
|
||||
capturer = _capturer_with_buffer(tp_size=2)
|
||||
topk = torch.arange(num_rows * 2, dtype=torch.int32).reshape(-1, 2)
|
||||
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=num_tokens))
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
|
||||
):
|
||||
capturer.capture(0, topk)
|
||||
get_tp_group.assert_not_called()
|
||||
assert torch.equal(capturer.device_buffer[:num_tokens, 0], topk[:num_tokens])
|
||||
assert (capturer.device_buffer[num_tokens:, 0] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens, num_rows", [(6, 2), (0, 1)])
|
||||
def test_single_dp_tp_capture_rejects_unexpected_batch_without_writing(
|
||||
num_tokens, num_rows
|
||||
):
|
||||
capturer = _capturer_with_buffer(tp_size=2)
|
||||
topk = torch.ones(num_rows, 2, dtype=torch.int32)
|
||||
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=num_tokens))
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
pytest.raises(AssertionError, match="unexpected topk_ids batch dim"),
|
||||
):
|
||||
capturer.capture(0, topk)
|
||||
assert (capturer.device_buffer == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
def test_capture_uses_pre_sp_size_after_model_shards_padding_mask(tp_rank):
|
||||
"""Model-level SP shortens the mask; capture still needs all three rows."""
|
||||
from vllm.models.common.ops import sequence_parallel as sp
|
||||
|
||||
capturer = _capturer_with_buffer(tp_size=2)
|
||||
expected = torch.arange(8, dtype=torch.int32).reshape(4, 2)
|
||||
ctx = _forward_context(
|
||||
is_padding=torch.zeros(3, dtype=torch.bool),
|
||||
batch_descriptor=BatchDescriptor(num_tokens=3),
|
||||
)
|
||||
with (
|
||||
patch.object(sp, "get_tensor_model_parallel_world_size", return_value=2),
|
||||
patch.object(sp, "get_tensor_model_parallel_rank", return_value=tp_rank),
|
||||
):
|
||||
ctx.is_padding = sp.sp_padding_mask(ctx.is_padding, torch.empty(3, 1))
|
||||
assert len(ctx.is_padding) == 2
|
||||
tp_group = Mock()
|
||||
tp_group.all_gather.return_value = expected
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
|
||||
):
|
||||
capturer.capture(0, expected[2 * tp_rank : 2 * (tp_rank + 1)])
|
||||
tp_group.all_gather.assert_called_once()
|
||||
torch.testing.assert_close(capturer.device_buffer[:3, 0], expected[:3])
|
||||
assert (capturer.device_buffer[3:, 0] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "dtype_name", "torch_dtype"),
|
||||
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
|
||||
@@ -359,7 +479,7 @@ def test_routed_experts_capturer_narrows_snapshot(output_dtype):
|
||||
capturer = _capturer_with_buffer(dtype=torch.int32)
|
||||
capturer.output_dtype = output_dtype
|
||||
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
|
||||
ctx = SimpleNamespace(dp_metadata=None)
|
||||
ctx = _forward_context()
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
|
||||
@@ -375,8 +495,8 @@ def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
|
||||
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
|
||||
capturer = _capturer_with_buffer(dp_rank=1)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
)
|
||||
# Concatenated order: rank0 rows then rank1 rows.
|
||||
topk = torch.tensor(
|
||||
@@ -392,8 +512,8 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
|
||||
"""N == token_num_per_dp: topk is already local to this DP rank."""
|
||||
capturer = _capturer_with_buffer(dp_rank=1)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
)
|
||||
topk = torch.tensor([[10, 11], [12, 13], [14, 15]], dtype=torch.int32)
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
@@ -404,8 +524,8 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
|
||||
def test_routed_experts_capturer_dp_ep_gathered_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(
|
||||
num_tokens_across_dp_cpu=num_tokens_dp,
|
||||
local_sizes=[2, 2, 1, 1],
|
||||
)
|
||||
@@ -421,13 +541,17 @@ def test_routed_experts_capturer_dp_ep_gathered_shards():
|
||||
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
@pytest.mark.parametrize("single_dp", [False, True])
|
||||
def test_routed_experts_capturer_sp_modular_gathers_tp_shards(single_dp):
|
||||
capturer = _capturer_with_buffer(dp_rank=0 if single_dp else 1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=(
|
||||
None if single_dp else DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
),
|
||||
batch_descriptor=BatchDescriptor(num_tokens=3),
|
||||
)
|
||||
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
|
||||
local_shard = torch.tensor([[10, 12], [11, 13]], dtype=torch.int32).t()
|
||||
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
|
||||
tp_group = Mock()
|
||||
tp_group.all_gather.return_value = gathered
|
||||
@@ -438,16 +562,83 @@ def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
|
||||
):
|
||||
capturer.capture(layer_id=0, topk_ids=local_shard)
|
||||
|
||||
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
|
||||
tp_group.all_gather.assert_called_once()
|
||||
gather_input = tp_group.all_gather.call_args.args[0]
|
||||
assert gather_input.is_contiguous()
|
||||
torch.testing.assert_close(gather_input, local_shard)
|
||||
assert tp_group.all_gather.call_args.kwargs == {"dim": 0}
|
||||
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_rejects_incomplete_dp_shard_groups():
|
||||
"""Invalid gather metadata must not silently select another rank's rows."""
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(
|
||||
num_tokens_across_dp_cpu=torch.tensor([3, 2]),
|
||||
local_sizes=[2, 2, 2],
|
||||
)
|
||||
)
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
pytest.raises(ValueError, match="not divisible by DP rank count"),
|
||||
):
|
||||
capturer.capture(0, torch.ones(6, 2, dtype=torch.int32))
|
||||
assert (capturer.device_buffer == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layer_id", [-1, 4])
|
||||
def test_routed_experts_capturer_rejects_layer_before_restoring_rows(layer_id):
|
||||
capturer = _capturer_with_buffer(tp_size=2)
|
||||
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=3))
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
|
||||
pytest.raises(IndexError, match="exceeds capture buffer"),
|
||||
):
|
||||
capturer.capture(layer_id, torch.ones(2, 2, dtype=torch.int32))
|
||||
get_tp_group.assert_not_called()
|
||||
assert (capturer.device_buffer == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("counts", "local_sizes", "start", "stop"),
|
||||
[
|
||||
([3, 0], [2, 2, 0, 0], 3, 3),
|
||||
([0, 3], None, 0, 3),
|
||||
([0, 0], [0, 0, 0, 0], 0, 0),
|
||||
([2, 2], [1, 1, 1, 1], 2, 4),
|
||||
],
|
||||
ids=["idle-rank", "only-active-rank", "all-idle", "unpadded-gather-tie"],
|
||||
)
|
||||
def test_routed_experts_capturer_dp_preserves_empty_and_unpadded_spans(
|
||||
counts, local_sizes, start, stop
|
||||
):
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
topk = torch.arange(sum(counts) * 2, dtype=torch.int32).reshape(-1, 2)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(
|
||||
num_tokens_across_dp_cpu=torch.tensor(counts), local_sizes=local_sizes
|
||||
)
|
||||
)
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
|
||||
):
|
||||
capturer.capture(0, topk)
|
||||
get_tp_group.assert_not_called()
|
||||
torch.testing.assert_close(
|
||||
capturer.device_buffer[: stop - start, 0], topk[start:stop]
|
||||
)
|
||||
assert (capturer.device_buffer[stop - start :, 0] == -1).all()
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
"""Mismatch between topk batch dim and DP layout: fail fast."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
ctx = _forward_context(
|
||||
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
)
|
||||
# total=5, local=2: n=1 matches neither naive (5) nor modular (2).
|
||||
topk = torch.tensor([[1, 2]], dtype=torch.int32)
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm fused KDA decode kernel must match the Triton chain it replaces.
|
||||
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
|
||||
|
||||
The fused kernel folds the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, and updates both the
|
||||
conv state and the recurrent state in place. Every one of those outputs is
|
||||
compared against the three-kernel path the AMD layer falls back to.
|
||||
Two fused paths fold the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, updating the conv
|
||||
state and the recurrent state in place:
|
||||
|
||||
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
|
||||
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
|
||||
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
|
||||
|
||||
Every output of each is compared against the unfused path the AMD layer falls
|
||||
back to, with the argument layout the layer itself uses.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +20,7 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_supported_arch() -> bool:
|
||||
def _hip_kernel_arch() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
@@ -22,10 +28,26 @@ def _on_supported_arch() -> bool:
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def _aiter_gluon_kda_available() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_supported_arch(),
|
||||
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
|
||||
)
|
||||
requires_hip_kernel_arch = pytest.mark.skipif(
|
||||
not _hip_kernel_arch(),
|
||||
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
|
||||
)
|
||||
requires_aiter_gluon_kda = pytest.mark.skipif(
|
||||
not _aiter_gluon_kda_available(),
|
||||
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
|
||||
)
|
||||
|
||||
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
|
||||
HEAD_DIM = 128
|
||||
@@ -41,7 +63,11 @@ def _requires_kernel() -> None:
|
||||
|
||||
|
||||
class KdaDecodeInputs:
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout."""
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
|
||||
|
||||
``num_spec`` > 0 sizes the conv state for speculative decoding
|
||||
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -49,6 +75,7 @@ class KdaDecodeInputs:
|
||||
num_heads: int,
|
||||
num_slots: int,
|
||||
seed: int = 0,
|
||||
num_spec: int = 0,
|
||||
) -> None:
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
@@ -70,9 +97,15 @@ class KdaDecodeInputs:
|
||||
for i in range(3)
|
||||
]
|
||||
)
|
||||
# SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it.
|
||||
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
|
||||
self.conv_state = (
|
||||
torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE)
|
||||
torch.randn(
|
||||
num_slots,
|
||||
CONV_WIDTH - 1 + num_spec,
|
||||
3 * dim,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
self.recurrent_state = (
|
||||
@@ -112,6 +145,12 @@ class KdaDecodeInputs:
|
||||
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
|
||||
return state.transpose(-1, -2)
|
||||
|
||||
def empty_out(self) -> torch.Tensor:
|
||||
num_tokens = self.mixed_qkv.shape[0]
|
||||
return torch.empty(
|
||||
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _gated_rmsnorm(
|
||||
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
@@ -196,6 +235,7 @@ def _run_fused(
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@pytest.mark.parametrize("num_heads", [12, 24, 96])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
@@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by state_indices may be written."""
|
||||
@@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_without_output_norm() -> None:
|
||||
"""Omitting the gate/norm pair returns the raw recurrent output."""
|
||||
@@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None:
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
@@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
"""The fused-decode weight copies must not depend on the weight loaders.
|
||||
|
||||
@@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
config, vllm_config, prefix="model.layers.0.self_attn"
|
||||
)
|
||||
assert layer.decode_conv1d_weight is not None
|
||||
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
|
||||
# kernel gets an fp32 norm copy.
|
||||
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
|
||||
with torch.no_grad():
|
||||
for param in layer.parameters():
|
||||
param.normal_()
|
||||
layer.decode_conv1d_weight.fill_(float("nan"))
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
if layer.decode_norm_weight is not None:
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
|
||||
process_weights_after_loading(
|
||||
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
|
||||
@@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
|
||||
)
|
||||
if layer.decode_norm_weight is not None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
assert not layer.decode_conv1d_weight.requires_grad
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# non-spec decode: fused_recurrent_kda_packed_decode
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_aiter_packed_decode(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
num_tokens = inp.mixed_qkv.shape[0]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv=inp.mixed_qkv,
|
||||
g=inp.g1[0],
|
||||
beta=inp.beta[0],
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
ssm_state_indices=inp.state_indices[:num_tokens],
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:num_tokens],
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[0, :num_tokens],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_matches_triton_chain(
|
||||
num_heads: int, num_tokens: int
|
||||
) -> None:
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
# The fused kernel keeps the recurrent output in fp32 through the norm,
|
||||
# while the Triton chain rounds it to BF16 in between.
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
# The conv state is a pure shift-and-append: it must be bit-exact.
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by the state indices may be written."""
|
||||
num_tokens, num_heads, num_slots = 4, 12, 9
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
|
||||
|
||||
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
touched = set(inp.state_indices.tolist())
|
||||
untouched = [slot for slot in range(num_slots) if slot not in touched]
|
||||
assert untouched, "test needs at least one unused slot"
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
|
||||
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
|
||||
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
|
||||
must do the same rather than read-modify-write slot 0 once per padded row.
|
||||
"""
|
||||
num_real, num_padded, num_heads = 3, 5, 12
|
||||
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
|
||||
inp.state_indices[num_real:] = 0
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
|
||||
)
|
||||
assert torch.isfinite(actual_out).all()
|
||||
# Slot 0 must be untouched by both paths.
|
||||
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SpecBatch:
|
||||
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
|
||||
|
||||
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
|
||||
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
|
||||
per-token slot table the GDN metadata hands the layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_lens: list[int],
|
||||
num_accepted: list[int],
|
||||
num_heads: int,
|
||||
num_spec: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
assert len(seq_lens) == len(num_accepted)
|
||||
assert max(seq_lens) <= num_spec + 1
|
||||
n_seqs = len(seq_lens)
|
||||
slots_per_seq = num_spec + 1
|
||||
num_slots = n_seqs * slots_per_seq + 2
|
||||
self.inp = KdaDecodeInputs(
|
||||
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
|
||||
)
|
||||
# Distinct slots per (request, token), shuffled, none at slot 0.
|
||||
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
|
||||
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
|
||||
self.query_start_loc = torch.tensor(
|
||||
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
self.num_accepted_tokens = torch.tensor(
|
||||
num_accepted, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
self.max_query_len = slots_per_seq
|
||||
|
||||
|
||||
def _run_spec_triton_chain(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
conv_out = causal_conv1d_update(
|
||||
inp.mixed_qkv,
|
||||
inp.conv_state_view(conv_state),
|
||||
inp.conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_state_indices=sb.state_indices[:, 0],
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
query_start_loc=sb.query_start_loc,
|
||||
max_query_len=sb.max_query_len,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(inp.mixed_qkv),
|
||||
)
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
|
||||
)
|
||||
core_attn_out, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
)
|
||||
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
def _run_aiter_spec(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM)
|
||||
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
|
||||
)
|
||||
n_spec = q.shape[1]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=inp.g1,
|
||||
beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:n_spec].unsqueeze(0),
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[:, :n_spec],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize(
|
||||
"num_spec, seq_lens, num_accepted",
|
||||
[
|
||||
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
|
||||
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
|
||||
(6, [7, 3], [1, 4]), # longer window, two requests
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_aiter_spec_decode_matches_triton_chain(
|
||||
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
|
||||
) -> None:
|
||||
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
|
||||
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
# Nothing outside the slot table may change.
|
||||
touched = set(sb.state_indices.flatten().tolist())
|
||||
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
|
||||
assert untouched
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
|
||||
@@ -75,6 +75,9 @@ AITER_MODEL_LIST = [
|
||||
"facebook/opt-125m", # opt
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-350m", # opt with projected token embeddings
|
||||
),
|
||||
pytest.param(
|
||||
"microsoft/phi-2", # phi
|
||||
marks=[pytest.mark.core_model, pytest.mark.slow_test],
|
||||
|
||||
@@ -70,6 +70,10 @@ def gemma3_chat_template(content: str) -> str:
|
||||
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
|
||||
|
||||
|
||||
def deepseek_vl2_chat_template(content: str) -> str:
|
||||
return f"<|User|>: {content}\n\n<|Assistant|>:"
|
||||
|
||||
|
||||
def ernie45_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
f"<|begin_of_sentence|>User: {content}"
|
||||
@@ -255,7 +259,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"attention_backend": (
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4"
|
||||
"ROCM_FLASHMLA_SPARSE_DSV41"
|
||||
if current_platform.is_rocm()
|
||||
else "FLASHMLA_SPARSE_DSV41"
|
||||
),
|
||||
@@ -356,6 +360,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
},
|
||||
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
|
||||
),
|
||||
"deepseek_vl2": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/deepseek-vl2-tiny",
|
||||
modalities=["image"],
|
||||
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
|
||||
vllm_runner_kwargs={
|
||||
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"gemma4": VitCudagraphTestConfig(
|
||||
model="google/gemma-4-E2B-it",
|
||||
image_prompt=(
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
|
||||
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
|
||||
|
||||
The runtime reads a question's options from the 255 code logits of its readout
|
||||
at the last prompt token. vLLM's classify head must give the same code logits
|
||||
for the same prompt tokens, and for an image prompt that vLLM processes itself.
|
||||
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
|
||||
weight file without an index, next to the readout file.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +16,7 @@ from transformers import AutoModel
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "vllm-sr/d3-edge"
|
||||
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
|
||||
STATE = (
|
||||
"The order arrived damaged yesterday. The customer has a receipt and asks "
|
||||
"for a replacement today."
|
||||
@@ -47,7 +49,7 @@ def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return torch.softmax(logits[:count].float(), dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL])
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
|
||||
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
|
||||
runtime = reference.runtime
|
||||
|
||||
@@ -70,20 +70,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
|
||||
backend: str, use_sp: bool
|
||||
) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
|
||||
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
@@ -98,7 +99,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
if backend == "nvidia":
|
||||
prev_block_output = kwargs["prev_block_output"]
|
||||
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
@@ -120,9 +123,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
|
||||
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
|
||||
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.qwen4_exp.amd import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
|
||||
else:
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
if hasattr(layer.ple, "start_prefetch"):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# ROCm all-gather requires contiguous inputs.
|
||||
output = tuple(
|
||||
sp_all_gather(x.contiguous())[:num_tokens] for x in output
|
||||
)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
def _check_compiled_sp_ops() -> None:
|
||||
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
|
||||
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
|
||||
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
|
||||
return gathered, reduced
|
||||
|
||||
# vLLM drops Dynamo guards, so one trace must serve every token count.
|
||||
compiled = torch.compile(
|
||||
collectives,
|
||||
fullgraph=True,
|
||||
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
|
||||
)
|
||||
traced = torch.randn(8, 16)
|
||||
torch._dynamo.mark_dynamic(traced, 0)
|
||||
compiled(traced)
|
||||
for num_tokens in (7, 3, 1):
|
||||
torch.manual_seed(num_tokens)
|
||||
x = torch.randn(num_tokens, 16)
|
||||
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
elif compiled_ops:
|
||||
_check_compiled_sp_ops()
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
|
||||
"""Padding in compiled SP collectives must not specialize on the traced size."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
|
||||
|
||||
@@ -462,7 +462,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
|
||||
"OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
|
||||
"OPTForCausalLM": _HfExamplesInfo(
|
||||
"facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
|
||||
"facebook/opt-125m",
|
||||
{"350m": "facebook/opt-350m", "1b": "facebook/opt-iml-max-1.3b"},
|
||||
),
|
||||
"OrionForCausalLM": _HfExamplesInfo(
|
||||
"OrionStarAI/Orion-14B-Chat", trust_remote_code=True
|
||||
@@ -731,7 +732,13 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"D3ForSequenceClassification": _HfExamplesInfo(
|
||||
"vllm-sr/d3-edge",
|
||||
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
|
||||
extras={
|
||||
"d3": "vllm-sr/d3",
|
||||
"d3-flash": "vllm-sr/d3-flash",
|
||||
"d3-mini": "vllm-sr/d3-mini",
|
||||
"d3-nano": "vllm-sr/d3-nano",
|
||||
"d3-lite": "vllm-sr/d3-lite",
|
||||
},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"GPT2ForSequenceClassification": _HfExamplesInfo(
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pcp_size,use_ep", [(1, False), (2, True), (2, False), (4, False)]
|
||||
)
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_pcp_gathers_routing_ids_once_after_local_input_preparation(
|
||||
pcp_size, use_ep, use_sp, monkeypatch
|
||||
):
|
||||
"""Embedding keeps local IDs; all decoder layers receive aligned routing IDs."""
|
||||
gather = Mock(side_effect=lambda ids, dim: torch.cat([ids] * pcp_size, dim=dim))
|
||||
monkeypatch.setattr(
|
||||
model_module, "get_pcp_group", lambda: SimpleNamespace(all_gather=gather)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(model_module, "is_forward_context_available", lambda: False)
|
||||
monkeypatch.setattr(model_module, "sp_shard", lambda tensor: tensor[::2])
|
||||
monkeypatch.setattr(
|
||||
model_module, "sp_all_gather", lambda tensor: tensor.repeat_interleave(2, dim=0)
|
||||
)
|
||||
monkeypatch.setattr(model_module, "hc_collapse_triton", lambda hidden, mix: hidden)
|
||||
embed = Mock(side_effect=lambda ids: ids[:, None].float())
|
||||
run_layers = Mock(
|
||||
side_effect=lambda layer_ids, hidden, *args: (hidden, None, None, None, hidden)
|
||||
)
|
||||
model = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=pcp_size, enable_expert_parallel=use_ep
|
||||
),
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=use_sp,
|
||||
engram_hash=None,
|
||||
embed_input_ids=embed,
|
||||
start_layer=0,
|
||||
decoder_replay_start=2,
|
||||
decoder_replay_layers=None,
|
||||
aux_hidden_state_layers=(),
|
||||
_mtp_hidden_buffer=None,
|
||||
_run_layers=run_layers,
|
||||
_collapse=lambda hidden, *args: hidden,
|
||||
norm=lambda hidden: hidden,
|
||||
)
|
||||
for offset in (0, 10):
|
||||
local_ids = torch.tensor([3, 7, 11, 15]) + offset
|
||||
model_module.DeepseekV4Model.forward(model, local_ids, torch.arange(4), None)
|
||||
assert embed.call_args.args[0] is local_ids
|
||||
routing_ids = local_ids[::2] if use_sp else local_ids
|
||||
needs_gather = pcp_size > 1 and not use_ep
|
||||
if needs_gather:
|
||||
torch.testing.assert_close(gather.call_args.args[0], routing_ids)
|
||||
routing_ids = routing_ids.repeat(pcp_size)
|
||||
torch.testing.assert_close(run_layers.call_args.args[3], routing_ids)
|
||||
assert gather.call_count == 2 * int(needs_gather)
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -16,6 +16,9 @@ from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
@@ -63,6 +66,7 @@ def state(monkeypatch):
|
||||
cfg.scheduler_config.max_num_seqs = 16
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.aux_output_config.enable_return_routed_experts = False
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
layers = DecoderReplayLayers(WINDOW, MagicMock(), MagicMock(), set(), "swa_first")
|
||||
@@ -172,6 +176,31 @@ def test_replay_batch_keeps_each_request_window(state):
|
||||
assert context.dp_metadata is None and context.is_padding is None
|
||||
|
||||
|
||||
def test_replay_context_restores_tp_routing_rows(state):
|
||||
"""Capture uses the compact replay batch's size, not the outer batch's."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, _ = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
context = replay.forward_context
|
||||
assert context.is_padding is None and context.dp_metadata is None
|
||||
count = len(REPLAY_ROWS)
|
||||
assert context.batch_descriptor.num_tokens == count
|
||||
capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
|
||||
capturer.dp_rank, capturer.tp_size = 0, 2
|
||||
capturer.device_buffer = torch.full((401, 1, 2), -1, device=DEVICE)
|
||||
expected = torch.tensor(REPLAY_ROWS, device=DEVICE)[:, None].expand(-1, 2)
|
||||
gathered = torch.cat((expected, torch.full((1, 2), -1, device=DEVICE)))
|
||||
with (
|
||||
override_forward_context(context),
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.routed_experts_capturer.get_tp_group"
|
||||
) as group,
|
||||
):
|
||||
group.return_value.all_gather.return_value = gathered
|
||||
capturer.capture(0, gathered[: (count + 1) // 2])
|
||||
torch.testing.assert_close(capturer.device_buffer[:count, 0], expected)
|
||||
assert (capturer.device_buffer[count:] == -1).all()
|
||||
|
||||
|
||||
def test_replay_batch_keeps_device_decode_boundaries(state):
|
||||
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
|
||||
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
|
||||
@@ -195,6 +224,27 @@ def test_prompt_logprobs_requests_keep_their_rows(state):
|
||||
assert build.batch.max_query_len == 300
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE])
|
||||
def test_aux_output_requests_keep_every_routing_row(state, cg_mode):
|
||||
"""R3 needs every prefill row even when final logits only need the window."""
|
||||
state.vllm_config.aux_output_config.enable_return_routed_experts = True
|
||||
for index in range(3):
|
||||
state.add_request(
|
||||
index,
|
||||
SimpleNamespace(
|
||||
replay_start=0,
|
||||
sampling_params=SimpleNamespace(prompt_logprobs=None),
|
||||
),
|
||||
)
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
state.decoder_replay_layers.trim_threshold = WINDOW + 1
|
||||
replay, _ = _prepare(state, batch, cg_mode)
|
||||
if cg_mode == CUDAGraphMode.NONE:
|
||||
assert replay is None
|
||||
else:
|
||||
torch.testing.assert_close(replay.rows, torch.arange(401, device=DEVICE))
|
||||
|
||||
|
||||
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
|
||||
"""Adaptive verification bounds the decodes' device-side query lengths
|
||||
above their CPU lengths; the replay batch keeps that bound."""
|
||||
|
||||
@@ -8,10 +8,8 @@ import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
DeepGemmSm100MegaMoEBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routed_experts_capturer,
|
||||
DeepGemmMegaMoEExperts,
|
||||
MegaMoEActivation,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
@@ -20,17 +18,14 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
DeepseekV4MegaMoEExperts,
|
||||
DeepseekV4MoE,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -40,11 +35,12 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sm100_mega_moe_backend(monkeypatch):
|
||||
def sm100_mega_moe_layout(monkeypatch):
|
||||
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
|
||||
lambda *args: DeepGemmSm100MegaMoEBackend(),
|
||||
"vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe."
|
||||
"check_deep_gemm_mega_moe_supported",
|
||||
lambda *args: None,
|
||||
)
|
||||
|
||||
|
||||
@@ -125,7 +121,8 @@ def test_deepseek_v41_moe_routes_without_hash_table(
|
||||
)
|
||||
assert moe.gate.tid2eid is None
|
||||
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
|
||||
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
|
||||
assert isinstance(moe.experts, DeepGemmMegaMoEExperts)
|
||||
assert moe.experts.activation == MegaMoEActivation("swiglu", clamp=10.0)
|
||||
assert moe.experts.w13_weight.shape == (num_experts, 256, 128)
|
||||
assert moe.experts.top_k == top_k
|
||||
assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6)
|
||||
@@ -151,10 +148,9 @@ def test_deepseek_v41_moe_routes_without_hash_table(
|
||||
|
||||
routed = {}
|
||||
|
||||
def check_routing(x, weights, ids, *, activation_clamp):
|
||||
def check_routing(x, weights, ids, shared_experts):
|
||||
routed["ids"], routed["weights"] = ids, weights
|
||||
assert activation_clamp == config.swiglu_limit
|
||||
return x.clone()
|
||||
return x + shared_experts(x)
|
||||
|
||||
monkeypatch.setattr(moe.experts, "forward", check_routing)
|
||||
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
|
||||
@@ -280,10 +276,10 @@ def test_deepseek_v4_mega_gate_hash_routing_correctness(
|
||||
dim=-1, keepdim=True
|
||||
)
|
||||
|
||||
def check_routing(x, weights, ids, *, activation_clamp):
|
||||
def check_routing(x, weights, ids, shared_experts):
|
||||
torch.testing.assert_close(ids, expected_ids)
|
||||
torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4)
|
||||
return x.clone()
|
||||
return x + shared_experts(x)
|
||||
|
||||
monkeypatch.setattr(moe.experts, "forward", check_routing)
|
||||
monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like)
|
||||
@@ -308,634 +304,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_kimi", [False, True])
|
||||
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
|
||||
experts_cls = DeepseekV4MegaMoEExperts
|
||||
if use_kimi:
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiK3MegaMoEExperts
|
||||
|
||||
experts_cls = KimiK3MegaMoEExperts
|
||||
|
||||
experts = experts_cls.__new__(experts_cls)
|
||||
torch.nn.Module.__init__(experts)
|
||||
if use_kimi:
|
||||
experts.synchronize_first_launch = lambda: None
|
||||
experts.prefix = "model.layers.3.ffn.experts"
|
||||
experts.max_num_tokens = 4
|
||||
experts.capture_fn = None
|
||||
experts.get_symm_buffer = lambda: object()
|
||||
experts.eplb_state = SimpleNamespace(
|
||||
logical_to_physical_map=torch.empty(1),
|
||||
expert_load_view=torch.empty(1),
|
||||
logical_replica_count=torch.empty(1),
|
||||
should_record_tensor=torch.empty(1),
|
||||
num_unpadded_tokens_tensors=None,
|
||||
)
|
||||
|
||||
topk_ids = torch.tensor([[1, 2], [3, 4]])
|
||||
captured: list[tuple[int, torch.Tensor]] = []
|
||||
bind_routed_experts_capturer(
|
||||
SimpleNamespace(modules=lambda: [experts]),
|
||||
SimpleNamespace(capture=lambda layer_id, ids: captured.append((layer_id, ids))),
|
||||
)
|
||||
|
||||
class MappingReached(Exception):
|
||||
pass
|
||||
|
||||
def map_ids(**kwargs):
|
||||
assert captured == [(3, topk_ids)]
|
||||
raise MappingReached
|
||||
|
||||
monkeypatch.setattr(
|
||||
f"{experts_cls.__module__}.eplb_map_to_physical_and_record",
|
||||
map_ids,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm", lambda: SimpleNamespace()
|
||||
)
|
||||
|
||||
with pytest.raises(MappingReached):
|
||||
experts(
|
||||
torch.empty(2, 8),
|
||||
torch.empty(2, 2),
|
||||
topk_ids,
|
||||
activation_clamp=None,
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
)
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=2,
|
||||
top_k=2,
|
||||
hidden_size=128,
|
||||
intermediate_size=128,
|
||||
)
|
||||
|
||||
nonlocal_weight = torch.ones(128, 64, dtype=torch.uint8)
|
||||
assert (
|
||||
experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
nonlocal_weight,
|
||||
"experts.w13_weight",
|
||||
shard_id="w1",
|
||||
expert_id=1,
|
||||
return_success=True,
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
w1 = torch.full((128, 64), 3, dtype=torch.uint8)
|
||||
w3 = torch.full((128, 64), 7, dtype=torch.uint8)
|
||||
w2 = torch.full((128, 64), 11, dtype=torch.uint8)
|
||||
|
||||
assert experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
w1,
|
||||
"experts.w13_weight",
|
||||
shard_id="w1",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
assert experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
w3,
|
||||
"experts.w13_weight",
|
||||
shard_id="w3",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
assert experts.weight_loader(
|
||||
experts.w2_weight,
|
||||
w2,
|
||||
"experts.w2_weight",
|
||||
shard_id="w2",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
|
||||
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
|
||||
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
|
||||
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
|
||||
assert torch.count_nonzero(experts.w13_weight[1]) == 0
|
||||
|
||||
|
||||
def _reference_mega_moe_layout(w13, s13, w2, s2):
|
||||
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
|
||||
|
||||
def interleave(t):
|
||||
g, n, k = t.shape
|
||||
halves = t.view(g, 2, n // 16, 8, k)
|
||||
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
|
||||
|
||||
def utccp(sf):
|
||||
g, mn, k = sf.shape
|
||||
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
|
||||
|
||||
return (
|
||||
interleave(w13),
|
||||
utccp(interleave(s13.view(torch.int32))),
|
||||
w2,
|
||||
utccp(s2.view(torch.int32)),
|
||||
)
|
||||
|
||||
|
||||
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
|
||||
return DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=num_local_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_logical_experts=num_logical_experts,
|
||||
)
|
||||
|
||||
|
||||
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
|
||||
hidden, inter = experts.hidden_size, experts.intermediate_size
|
||||
shapes = {
|
||||
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
|
||||
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
|
||||
"w2_weight": (num_logical_experts, hidden, inter // 2),
|
||||
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
|
||||
}
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
ckpt = {
|
||||
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
for name, shape in shapes.items()
|
||||
}
|
||||
shards = []
|
||||
for expert_id in range(num_logical_experts):
|
||||
for name, tensor in ckpt.items():
|
||||
if name.startswith("w2_"):
|
||||
shards.append((name, tensor[expert_id], "w2", expert_id))
|
||||
else:
|
||||
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
|
||||
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
|
||||
for i in torch.randperm(len(shards), generator=generator).tolist():
|
||||
name, loaded_weight, shard_id, expert_id = shards[i]
|
||||
if name.endswith("scale"):
|
||||
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
|
||||
assert experts.weight_loader(
|
||||
getattr(experts, name),
|
||||
loaded_weight,
|
||||
f"experts.{name}",
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
return ckpt
|
||||
|
||||
|
||||
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
|
||||
physical = torch.arange(num_local_experts) % num_logical_experts
|
||||
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
|
||||
expected = _reference_mega_moe_layout(*ckpt.values())
|
||||
if is_deep_gemm_supported():
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
|
||||
hidden, inter = w2.shape[1], w2.shape[2] * 2
|
||||
scales = [
|
||||
deep_gemm.transform_sf_into_required_layout(
|
||||
(sf.to(torch.int32) << 23).view(torch.float32),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
|
||||
]
|
||||
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
|
||||
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
|
||||
)
|
||||
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
|
||||
assert torch.equal(actual.cpu().view(reference.dtype), reference)
|
||||
return expected
|
||||
|
||||
|
||||
def _assert_mega_moe_layout(experts, expected):
|
||||
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
|
||||
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
|
||||
# DeepGEMM's MN-major scale strides.
|
||||
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
|
||||
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
|
||||
"""A second load must update the live kernel tensors without reallocating."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
|
||||
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
|
||||
data_ptrs = [param.data_ptr() for param in experts.parameters()]
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
|
||||
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate_size", [512, 2304])
|
||||
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
|
||||
"""Keep native widths so V4.1 routed and shared experts can fuse."""
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate_size,
|
||||
)
|
||||
|
||||
assert experts.intermediate_size == intermediate_size
|
||||
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
|
||||
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
|
||||
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
|
||||
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
|
||||
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for actual, param in zip(transformed, experts.parameters()):
|
||||
assert actual.data_ptr() == param.data_ptr()
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
|
||||
"""Shared expert shards land directly in the MegaMoE kernel layout."""
|
||||
monkeypatch.setattr(
|
||||
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
|
||||
)
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_shared_experts=1,
|
||||
)
|
||||
assert experts.has_fused_shared_experts
|
||||
hidden, inter = 256, 384
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
|
||||
def rand(*shape):
|
||||
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
|
||||
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
|
||||
s1, s3, s2 = (
|
||||
rand(inter, hidden // 32),
|
||||
rand(inter, hidden // 32),
|
||||
rand(hidden, inter // 32),
|
||||
)
|
||||
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
|
||||
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
|
||||
for name, data in [
|
||||
("shared_experts.w1.weight", w1.view(fp8)),
|
||||
("shared_experts.w3.weight", w3.view(fp8)),
|
||||
("shared_experts.down_proj.weight", w2.view(fp8)),
|
||||
("shared_experts.w1.weight_scale", s1.view(e8m0)),
|
||||
("shared_experts.w3.weight_scale", s3.view(e8m0)),
|
||||
("shared_experts.w2.weight_scale", s2.view(e8m0)),
|
||||
]:
|
||||
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
|
||||
# Weight tracking checks this name against the registered parameters.
|
||||
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
|
||||
|
||||
def l1_halves():
|
||||
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
|
||||
|
||||
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
|
||||
expected = _reference_mega_moe_layout(
|
||||
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
|
||||
)
|
||||
actual = (
|
||||
experts.shared_l1_weight,
|
||||
experts.shared_l1_weight_scale,
|
||||
experts.shared_l2_weight,
|
||||
experts.shared_l2_weight_scale,
|
||||
)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference[0])
|
||||
# A reload of one shard overwrites in place and leaves the other half alone.
|
||||
ptr = experts.shared_l1_weight.data_ptr()
|
||||
w1b = rand(inter, hidden)
|
||||
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
|
||||
assert experts.shared_l1_weight.data_ptr() == ptr
|
||||
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
|
||||
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
|
||||
with pytest.raises(ValueError):
|
||||
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
|
||||
|
||||
|
||||
def _fake_deep_gemm(with_shared: bool):
|
||||
def old_buffer(group): ...
|
||||
|
||||
def old_kernel(y): ...
|
||||
|
||||
def new_buffer(group, num_shared_experts=0): ...
|
||||
|
||||
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
|
||||
|
||||
return SimpleNamespace(
|
||||
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
|
||||
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
|
||||
get_block_m_for_mega_moe=lambda *args: 0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
|
||||
[
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
|
||||
(torch.bfloat16, None, None, True, False),
|
||||
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
|
||||
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
|
||||
):
|
||||
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
|
||||
)
|
||||
|
||||
def linear():
|
||||
scale = (
|
||||
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
|
||||
)
|
||||
|
||||
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=128,
|
||||
num_shared_experts=1,
|
||||
shared_experts=shared,
|
||||
)
|
||||
|
||||
assert experts.has_fused_shared_experts == fused
|
||||
assert hasattr(experts, "shared_l1_weight") == fused
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fused", [False, True])
|
||||
def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
monkeypatch, fused
|
||||
):
|
||||
class FakeGate(torch.nn.Module):
|
||||
weight = torch.empty(2, 128)
|
||||
tid2eid = None
|
||||
e_score_correction_bias = None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return torch.empty(hidden_states.shape[0], 2), None
|
||||
|
||||
class FakeExperts(torch.nn.Module):
|
||||
has_fused_shared_experts = fused
|
||||
backend = DeepGemmSm100MegaMoEBackend()
|
||||
|
||||
def forward(self, hidden_states, *args, **kwargs):
|
||||
return torch.ones_like(hidden_states)
|
||||
|
||||
class FakeSharedExperts(torch.nn.Module):
|
||||
calls = 0
|
||||
|
||||
def forward(self, hidden_states):
|
||||
self.calls += 1
|
||||
return torch.full_like(hidden_states, 2)
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = None if fused else FakeSharedExperts()
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = 1
|
||||
moe.renormalize = True
|
||||
moe.hash_indices_dtype = torch.int64
|
||||
moe.routed_scaling_factor = 1.0
|
||||
moe.swiglu_limit = 10.0
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.fused_topk_bias",
|
||||
lambda **kwargs: (
|
||||
torch.ones(kwargs["hidden_states"].shape[0], 1),
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 1, dtype=torch.int64),
|
||||
),
|
||||
)
|
||||
|
||||
output = moe(torch.zeros(2, 128))
|
||||
|
||||
expected = 1 if fused else 3
|
||||
assert torch.all(output == expected)
|
||||
if not fused:
|
||||
assert moe.shared_experts.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 5120])
|
||||
def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(
|
||||
num_tokens, hidden_size
|
||||
):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
top_k = 8
|
||||
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(0)
|
||||
hidden_states = (
|
||||
torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
* 17.0
|
||||
).to(torch.bfloat16)
|
||||
hidden_states[0, :32] = 0
|
||||
hidden_states[1, 32:64] = 1.0e-6
|
||||
hidden_states[2, 64:96] = -1.0e-6
|
||||
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
ref_topk_idx = topk_ids.to(torch.int64)
|
||||
ref_topk_weights = topk_weights.clone()
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_topk_idx = torch.empty_like(ref_topk_idx)
|
||||
fused_topk_weights = torch.empty_like(ref_topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_topk_idx, ref_topk_idx)
|
||||
assert torch.equal(
|
||||
fused_topk_weights.view(torch.uint8),
|
||||
ref_topk_weights.view(torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared_block_m", [8, 32, 96, 128, 192])
|
||||
def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_tokens = shared_block_m + 7
|
||||
hidden_size = 256
|
||||
top_k = 8
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(shared_block_m)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=generator,
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
aligned_block_m = ((shared_block_m + 127) // 128) * 128
|
||||
num_shared_rows = ((num_tokens + shared_block_m - 1) // shared_block_m) * (
|
||||
aligned_block_m
|
||||
)
|
||||
ref_shared_x_sf = torch.zeros(
|
||||
num_shared_rows,
|
||||
hidden_size // 128,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
for token_id in range(num_tokens):
|
||||
m_in_block = token_id % shared_block_m
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
shared_row = token_id // shared_block_m * aligned_block_m + transposed_m
|
||||
ref_shared_x_sf[shared_row].copy_(ref_x_sf[token_id])
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_shared_storage = torch.full(
|
||||
(hidden_size // 128, num_shared_rows),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
fused_shared_x_sf = fused_shared_storage.t()
|
||||
fused_topk_idx = torch.empty_like(topk_ids, dtype=torch.int64)
|
||||
fused_topk_weights = torch.empty_like(topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
shared_x_sf=fused_shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
populated = ref_shared_x_sf != 0
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated])
|
||||
assert torch.all(fused_shared_x_sf[~populated] == -1)
|
||||
|
||||
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
|
||||
"""The loader invokes the model-level PWAL hook for every load format,
|
||||
so it must finalize mhc broadcast weights to cover dummy load, which
|
||||
@@ -1040,85 +408,3 @@ def test_dspark_draft_registers_mixture_of_experts(
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize("nonfinite_padding", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [7, 65, 135])
|
||||
def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(
|
||||
nonfinite_padding, num_tokens
|
||||
):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
hidden_size = 256
|
||||
top_k = 8
|
||||
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(1)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=generator,
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool)
|
||||
is_padding[1::3] = True
|
||||
is_padding[-1] = True
|
||||
if nonfinite_padding:
|
||||
topk_weights[is_padding] = float("nan")
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
ref_topk_idx = topk_ids.to(torch.int64)
|
||||
ref_topk_idx[is_padding] = -1
|
||||
ref_topk_weights = topk_weights.clone()
|
||||
ref_topk_weights[is_padding] = 0.0
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_topk_idx = torch.empty_like(ref_topk_idx)
|
||||
fused_topk_weights = torch.empty_like(ref_topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_topk_idx, ref_topk_idx)
|
||||
assert torch.equal(
|
||||
fused_topk_weights.view(torch.uint8),
|
||||
ref_topk_weights.view(torch.uint8),
|
||||
)
|
||||
|
||||
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Qwen4ExpMTP", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForCausalLM", True),
|
||||
("Qwen4ExpForConditionalGeneration", True),
|
||||
("Qwen4ExpMTP", True),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -161,12 +161,15 @@ class TestReasoningEndTokenIds:
|
||||
)
|
||||
engine = _make_engine(cfg)
|
||||
assert engine.reasoning_end_token_ids == {201, 202}
|
||||
assert engine.implicit_reasoning_end_token_ids == {202}
|
||||
assert engine.find_reasoning_end([5, 201]) == ReasoningEnd(1, False)
|
||||
assert engine.find_reasoning_end([5, 202]) == ReasoningEnd(1, True)
|
||||
|
||||
def test_transition_staying_in_reasoning_is_ignored(self):
|
||||
cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ()))
|
||||
assert _make_engine(cfg).reasoning_end_token_ids == {201}
|
||||
engine = _make_engine(cfg)
|
||||
assert engine.reasoning_end_token_ids == {201}
|
||||
assert engine.implicit_reasoning_end_token_ids == frozenset()
|
||||
|
||||
def test_unreported_exit_fails_closed(self):
|
||||
cfg = _with_reasoning_exits(
|
||||
|
||||
@@ -6,11 +6,16 @@ from vllm import ModelRegistry
|
||||
|
||||
def register():
|
||||
# Test directly passing the model
|
||||
from .my_opt import MyOPTForCausalLM
|
||||
from .my_opt import MyOPTForCausalLM, MyOPTRawInputTokensForCausalLM
|
||||
|
||||
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
|
||||
|
||||
if "MyOPTRawInputTokensForCausalLM" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model(
|
||||
"MyOPTRawInputTokensForCausalLM", MyOPTRawInputTokensForCausalLM
|
||||
)
|
||||
|
||||
# Test passing lazy model
|
||||
if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.opt import OPTForCausalLM
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class MyOPTForCausalLM(OPTForCausalLM):
|
||||
@@ -15,3 +16,21 @@ class MyOPTForCausalLM(OPTForCausalLM):
|
||||
logits.zero_()
|
||||
logits[:, 0] += 1.0
|
||||
return logits
|
||||
|
||||
|
||||
class MyOPTRawInputTokensForCausalLM(MyOPTForCausalLM):
|
||||
# Like models whose decoder layers read token ids, this model needs
|
||||
# input_ids on every pipeline-parallel stage, not only the first.
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
assert input_ids is not None, "requires_raw_input_tokens model got no input_ids"
|
||||
return super().forward(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
assert chatml_jinja_path.exists()
|
||||
|
||||
|
||||
def run_and_test_dummy_opt_api_server(model, tp=1):
|
||||
def run_and_test_dummy_opt_api_server(model, tp=1, extra_args=(), env_dict=None):
|
||||
# the model is registered through the plugin
|
||||
server_args = [
|
||||
"--gpu-memory-utilization",
|
||||
@@ -20,8 +20,9 @@ def run_and_test_dummy_opt_api_server(model, tp=1):
|
||||
"dummy",
|
||||
"-tp",
|
||||
f"{tp}",
|
||||
*extra_args,
|
||||
]
|
||||
with RemoteOpenAIServer(model, server_args) as server:
|
||||
with RemoteOpenAIServer(model, server_args, env_dict=env_dict) as server:
|
||||
client = server.get_client()
|
||||
completion = client.chat.completions.create(
|
||||
model=model,
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (has_humming() and current_platform.is_cuda()),
|
||||
reason="requires humming and CUDA",
|
||||
)
|
||||
|
||||
|
||||
def test_humming_input_quant_compiles_fullgraph():
|
||||
"""Humming's forward helpers look up ``DeviceInfo`` on every call; vLLM
|
||||
traces them with ``fullgraph=True`` (vllm-project/humming#147)."""
|
||||
import vllm.utils.humming as hm
|
||||
|
||||
weight_schema = hm.HummingWeightSchema(b_dtype="int4", weight_scale_group_size=128)
|
||||
input_schema = hm.HummingInputSchema(a_dtype="int8")
|
||||
from humming.transform import prepare_layer_config
|
||||
|
||||
config = prepare_layer_config(
|
||||
shape_n=256,
|
||||
shape_k=256,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
torch_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
x = torch.randn(16, 256, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def quant(t: torch.Tensor):
|
||||
return hm.may_process_input(config, t)
|
||||
|
||||
expected = quant(x)
|
||||
actual = torch.compile(quant, fullgraph=True)(x)
|
||||
for a, e in zip(actual, expected):
|
||||
if e is None:
|
||||
assert a is None
|
||||
else:
|
||||
torch.testing.assert_close(a, e)
|
||||
@@ -105,6 +105,7 @@ def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicM
|
||||
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
|
||||
# and dtype, so the mock must return concrete values.
|
||||
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
|
||||
mc.hf_text_config = object()
|
||||
mc.dtype = _MOCK_DTYPE
|
||||
return mc
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ class MockModelConfig:
|
||||
tokenizer_revision = None
|
||||
tokenizer_mode = "auto"
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = hf_config
|
||||
encoder_config: dict[str, Any] | None = None
|
||||
enable_prompt_embeds: bool = True
|
||||
skip_tokenizer_init: bool = False
|
||||
|
||||
@@ -65,6 +65,40 @@ def _saved_tensor(tensor: torch.Tensor) -> bytes:
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, hidden_size, input_size, buffer_size",
|
||||
[
|
||||
("facebook/opt-350m", 1024, 512, 512),
|
||||
("facebook/opt-125m", 768, 768, 768),
|
||||
("openai/clip-vit-base-patch32", 512, 512, 1024),
|
||||
("google/siglip-base-patch16-224", 768, 768, 1024),
|
||||
],
|
||||
)
|
||||
def test_prompt_embeds_input_width(model, hidden_size, input_size, buffer_size):
|
||||
"""OPT uses its token width; dual encoders retain their text input width."""
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
config = ModelConfig(
|
||||
model=model,
|
||||
dtype="float32",
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
if "opt-" not in model:
|
||||
config.hf_text_config.projection_dim = buffer_size
|
||||
assert config.get_hidden_size() == hidden_size
|
||||
assert config.get_inputs_embeds_size() == buffer_size
|
||||
|
||||
tensor = torch.randn(4, input_size)
|
||||
loaded = safe_load_prompt_embeds(config, _encode(_saved_tensor(tensor)))
|
||||
torch.testing.assert_close(loaded, tensor)
|
||||
|
||||
wrong_size = hidden_size if hidden_size != input_size else input_size + 1
|
||||
with pytest.raises(VLLMValidationError, match="does not match"):
|
||||
safe_load_prompt_embeds(
|
||||
config, _encode(_saved_tensor(torch.randn(4, wrong_size)))
|
||||
)
|
||||
|
||||
|
||||
UNPARSABLE_PAYLOADS = [
|
||||
pytest.param(b"", id="empty"),
|
||||
pytest.param(b"\x80", id="single-byte"),
|
||||
|
||||
+30
-13
@@ -2132,8 +2132,8 @@ def test_draft_parallel_config_preserves_ep_without_model(
|
||||
[
|
||||
pytest.param("eagle3", False, 0, id="eagle3"),
|
||||
pytest.param("eagle3", True, 7, id="p-eagle"),
|
||||
pytest.param("dflash", True, 8, id="dflash"),
|
||||
pytest.param("dspark", True, 7, id="dspark"),
|
||||
pytest.param("dflash", True, 0, id="dflash"),
|
||||
pytest.param("dspark", True, 0, id="dspark"),
|
||||
pytest.param("mtp", False, 0, id="mtp"),
|
||||
pytest.param("ngram", False, 0, id="ngram"),
|
||||
pytest.param("draft_model", False, 1, id="draft-model"),
|
||||
@@ -3769,34 +3769,48 @@ def test_dual_key_gumbel_requires_probabilistic_drafting():
|
||||
config._check_supports_watermarking()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"])
|
||||
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method):
|
||||
@pytest.mark.parametrize(
|
||||
("method", "parallel_drafting"),
|
||||
[
|
||||
("eagle", False),
|
||||
("eagle3", False),
|
||||
("mtp", False),
|
||||
("dspark", True),
|
||||
("dflash", True),
|
||||
],
|
||||
)
|
||||
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(
|
||||
method, parallel_drafting
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method=method,
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
parallel_drafting=parallel_drafting,
|
||||
)
|
||||
|
||||
config._check_supports_watermarking()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_supports_dspark():
|
||||
@pytest.mark.parametrize("architecture", ["DFlash2DraftModel", "LiLiCorrDraftModel"])
|
||||
def test_dual_key_gumbel_rejects_dflash_candidate_drafts(architecture):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="dspark",
|
||||
method="dflash",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=True,
|
||||
draft_model_config=SimpleNamespace(architectures=[architecture]),
|
||||
)
|
||||
|
||||
config._check_supports_watermarking()
|
||||
with pytest.raises(ValueError, match="candidate-head drafters"):
|
||||
config._check_supports_watermarking()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
|
||||
def test_dual_key_gumbel_rejects_unsupported_speculative_method():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
@@ -3806,7 +3820,7 @@ def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="autoregressive model-based"):
|
||||
with pytest.raises(ValueError, match="Watermarking supports only"):
|
||||
config._check_supports_watermarking()
|
||||
|
||||
|
||||
@@ -4167,7 +4181,7 @@ def test_revision_resolved_for_model(mock_resolve):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,supported,is_cuda,tp_size,pp_size,error",
|
||||
"enabled,supported,cuda_alike,tp_size,pp_size,error",
|
||||
[
|
||||
(True, True, True, 2, 1, None),
|
||||
(True, False, True, 2, 1, "not supported for this model"),
|
||||
@@ -4178,7 +4192,7 @@ def test_revision_resolved_for_model(mock_resolve):
|
||||
(False, False, False, 2, 1, None),
|
||||
],
|
||||
)
|
||||
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
|
||||
def test_hc_sp_model_support(enabled, supported, cuda_alike, tp_size, pp_size, error):
|
||||
"""Validate active HC SP configurations and ignore the flag with TP=1."""
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
|
||||
@@ -4196,7 +4210,10 @@ def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, erro
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
with (
|
||||
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
|
||||
patch(
|
||||
"vllm.config.model.current_platform.is_cuda_alike",
|
||||
return_value=cuda_alike,
|
||||
),
|
||||
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
|
||||
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
|
||||
):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import _get_protocol_attrs # type: ignore
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -15,7 +16,11 @@ from transformers import (
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import resolve_tokenizer_args
|
||||
from vllm.tokenizers.registry import (
|
||||
cached_tokenizer_from_config,
|
||||
resolve_tokenizer_args,
|
||||
tokenizer_args_from_config,
|
||||
)
|
||||
|
||||
|
||||
def _get_missing_attrs(obj: object, target: type):
|
||||
@@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
|
||||
@pytest.mark.parametrize(
|
||||
"code_revision, error_match",
|
||||
[
|
||||
pytest.param("never", "not a valid git identifier", id="unknown-revision"),
|
||||
pytest.param(
|
||||
"never",
|
||||
"not a valid git identifier|couldn't find them in the cached files",
|
||||
id="unknown-revision",
|
||||
),
|
||||
pytest.param(
|
||||
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||
r"tokenization_fast\.py",
|
||||
@@ -143,6 +152,141 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"code_revision",
|
||||
[
|
||||
None,
|
||||
"c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2",
|
||||
"2efddce40dddaccd37bae208c3c7ca66dbedf68a",
|
||||
],
|
||||
)
|
||||
def test_tokenizer_config_code_revision(code_revision):
|
||||
"""Load real remote config code at its pin, not the data revision."""
|
||||
revision = "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2"
|
||||
|
||||
with patch(
|
||||
"vllm.tokenizers.registry.CachedHfTokenizer.from_pretrained",
|
||||
return_value=SimpleNamespace(is_fast=True),
|
||||
) as load_tokenizer:
|
||||
get_tokenizer(
|
||||
"hf-internal-testing/test_dynamic_model",
|
||||
tokenizer_mode="hf",
|
||||
trust_remote_code=True,
|
||||
revision=revision,
|
||||
code_revision=code_revision,
|
||||
)
|
||||
|
||||
config = load_tokenizer.call_args.kwargs["config"]
|
||||
assert (code_revision or revision) in type(config).__module__.split(".")
|
||||
assert config.new_hidden_size == 12
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model, code_revision, kwargs, error_match",
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
"never",
|
||||
{},
|
||||
"not a valid git identifier|couldn't find them in the cached files",
|
||||
id="same-repo",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||
{},
|
||||
r"tokenization_fast\.py",
|
||||
id="same-repo-missing-code",
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-125m",
|
||||
"27dcfa74d334bc871f3234de431e71c6eeba5dd6",
|
||||
{},
|
||||
None,
|
||||
id="separate-repo",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
"never",
|
||||
{"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
|
||||
None,
|
||||
id="explicit-revision",
|
||||
),
|
||||
pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
|
||||
],
|
||||
)
|
||||
def test_config_tokenizer_code_revision(
|
||||
loader, model, code_revision, kwargs, error_match
|
||||
):
|
||||
"""Only same-repo tokenizers inherit model code revisions; overrides win."""
|
||||
revision = "08a35780b7a78794436d8654aafa26ee0165c149"
|
||||
repo = "hf-internal-testing/test_dynamic_tokenizer"
|
||||
config = SimpleNamespace(
|
||||
model=model or repo,
|
||||
tokenizer=repo,
|
||||
runner_type="generate",
|
||||
tokenizer_mode="hf",
|
||||
tokenizer_revision=revision,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
|
||||
def load():
|
||||
result = loader(config, **kwargs)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, resolved_kwargs = result
|
||||
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||
return result
|
||||
|
||||
if error_match is not None:
|
||||
with pytest.raises(OSError, match=error_match):
|
||||
load()
|
||||
else:
|
||||
tokenizer = load()
|
||||
reference = AutoTokenizer.from_pretrained(
|
||||
repo,
|
||||
revision=revision,
|
||||
code_revision=kwargs.get("code_revision"),
|
||||
trust_remote_code=True,
|
||||
)
|
||||
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"])
|
||||
@pytest.mark.parametrize("code_revision", [None, "never"])
|
||||
def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision):
|
||||
"""Config loading must not pass HF-only revisions to the Mistral backend."""
|
||||
repo = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||
config = SimpleNamespace(
|
||||
model=repo,
|
||||
tokenizer=repo,
|
||||
runner_type="generate",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
tokenizer = loader(config)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, kwargs = tokenizer
|
||||
tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs)
|
||||
|
||||
reference = get_tokenizer(repo, tokenizer_mode="mistral")
|
||||
assert isinstance(tokenizer, MistralTokenizer)
|
||||
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||
|
||||
|
||||
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||
with patch.object(
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.tokenizers.detokenizer_utils import (
|
||||
)
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.protocol import TokenizerLike
|
||||
from vllm.transformers_utils.repo_utils import with_retry
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.detokenizer import (
|
||||
FastIncrementalDetokenizer,
|
||||
@@ -95,10 +96,13 @@ def _run_incremental_decode(
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer(tokenizer_name):
|
||||
return (
|
||||
MistralTokenizer.from_pretrained(tokenizer_name)
|
||||
if "mistral" in tokenizer_name
|
||||
else AutoTokenizer.from_pretrained(tokenizer_name)
|
||||
return with_retry(
|
||||
lambda: (
|
||||
MistralTokenizer.from_pretrained(tokenizer_name)
|
||||
if "mistral" in tokenizer_name
|
||||
else AutoTokenizer.from_pretrained(tokenizer_name)
|
||||
),
|
||||
log_msg=f"Error loading tokenizer {tokenizer_name}",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import pytest
|
||||
from transformers import AutoConfig, Qwen3_5MoeConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers import TokenizerLike, registry
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
@@ -17,6 +17,7 @@ from vllm.tokenizers.registry import (
|
||||
cached_tokenizer_from_config,
|
||||
get_tokenizer,
|
||||
resolve_tokenizer_args,
|
||||
tokenizer_args_from_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
||||
)
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
skip_tokenizer_init=False,
|
||||
tokenizer=str(tmp_path),
|
||||
runner_type="generate",
|
||||
tokenizer_mode="hf",
|
||||
tokenizer_revision=None,
|
||||
code_revision=None,
|
||||
trust_remote_code=True,
|
||||
hf_config=Qwen3_5MoeConfig(),
|
||||
)
|
||||
@@ -164,3 +167,117 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
||||
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
||||
if registered_config is not None:
|
||||
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config_tokenizer_backend(monkeypatch):
|
||||
"""Keep config helpers and dispatch real; replace Hub/backend loading."""
|
||||
calls = []
|
||||
|
||||
class Backend:
|
||||
@classmethod
|
||||
def from_pretrained(cls, name, *args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(is_fast=True)
|
||||
|
||||
monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None)
|
||||
monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True)
|
||||
monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True)
|
||||
monkeypatch.setattr(
|
||||
registry, "get_non_weight_snapshot_path", lambda name, **kw: name
|
||||
)
|
||||
monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend)
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
yield calls
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
|
||||
|
||||
def _load_config_tokenizer(loader, config, **kwargs):
|
||||
result = loader(config, **kwargs)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, resolved_kwargs = result
|
||||
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("mode", ["mistral", "auto"])
|
||||
@pytest.mark.parametrize("code_revision", [None, "model-code"])
|
||||
@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}])
|
||||
def test_config_mistral_does_not_forward_code_revision(
|
||||
loader, mode, code_revision, overrides, config_tokenizer_backend
|
||||
):
|
||||
"""HF code revisions must not reach MistralCommonBackend's strict kwargs."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode=mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
_load_config_tokenizer(loader, config, **overrides)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert "code_revision" not in config_tokenizer_backend[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("missing", [False, True])
|
||||
def test_config_tokenizer_without_code_revision(
|
||||
loader, missing, config_tokenizer_backend
|
||||
):
|
||||
"""Default or minimal configs must not introduce an HF-only plugin kwarg."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode="test_tokenizer",
|
||||
tokenizer_revision=None,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
if not missing:
|
||||
config.code_revision = None
|
||||
_load_config_tokenizer(loader, config)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert "code_revision" not in config_tokenizer_backend[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"])
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
({}, "model-code"),
|
||||
({"code_revision": None}, None),
|
||||
({"code_revision": "override"}, "override"),
|
||||
],
|
||||
)
|
||||
def test_config_hf_backed_tokenizer_preserves_code_revision(
|
||||
loader, mode, overrides, expected, config_tokenizer_backend
|
||||
):
|
||||
"""Filtering Mistral kwargs must preserve HF wrappers and caller overrides."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode=mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision="model-code",
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
_load_config_tokenizer(loader, config, **overrides)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert config_tokenizer_backend[0]["code_revision"] == expected
|
||||
|
||||
@@ -244,6 +244,22 @@ class TestStreamingExtraction:
|
||||
assert tcs[0]["name"] == "get_weather"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
['{"value": -2e-3}', '{"value": 1.5e3}', '{"values": [1.5e3, -2e-3]}'],
|
||||
)
|
||||
def test_streaming_scientific_notation(self, parser, mock_request, arguments):
|
||||
model_output = (
|
||||
f'{TOOL_CALLS_PREFIX}[{{"calculate": {arguments}}}]{TOOL_CALLS_SUFFIX}'
|
||||
)
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, list(model_output))
|
||||
tool_calls = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["name"] == "calculate"
|
||||
assert json.loads(tool_calls[0]["arguments"]) == json.loads(arguments)
|
||||
|
||||
def test_streaming_missing_tool_suffix(self, parser, mock_request):
|
||||
chunks = [
|
||||
"<|tools_prefix|>",
|
||||
|
||||
@@ -763,7 +763,7 @@ def test_flashinfer_cross_layer_layout(
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec=BATCH_SPECS[batch_spec_name],
|
||||
model="meta-llama/Meta-Llama-3-8B",
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
backend_to_test=[AttentionBackendEnum.FLASHINFER],
|
||||
mask_mod=causal_mask_mod,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
@@ -786,7 +786,7 @@ def test_flashinfer_cross_layer_layout(
|
||||
"single_prefill",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
||||
def test_causal_backend_correctness(
|
||||
@@ -1582,7 +1582,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
|
||||
_test_backend_correctness(
|
||||
BATCH_SPECS["small_prefill"],
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
use_sinks=True,
|
||||
@@ -1629,7 +1629,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
|
||||
batch_spec = BATCH_SPECS["small_decode"]
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="meta-llama/Meta-Llama-3-8B",
|
||||
model_name="Qwen/Qwen3-0.6B",
|
||||
max_model_len=max(batch_spec.seq_lens),
|
||||
block_size=16,
|
||||
)
|
||||
@@ -1691,7 +1691,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec,
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
)
|
||||
@@ -1994,7 +1994,7 @@ if current_platform.is_rocm():
|
||||
"mixed_small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
||||
def test_non_causal_backend_correctness(
|
||||
default_vllm_config,
|
||||
batch_spec_name: str,
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
|
||||
@@ -126,7 +126,7 @@ def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
|
||||
|
||||
for builder_cls in (
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
|
||||
|
||||
@@ -529,7 +529,7 @@ def make_builder(
|
||||
window: int = WINDOW,
|
||||
max_image_tokens: int = MAX_IMG,
|
||||
max_num_batched_tokens: int = 64,
|
||||
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||
model_name: str = "Qwen/Qwen3-0.6B",
|
||||
builder_cls: type[
|
||||
DeepseekSparseSWAMetadataBuilder
|
||||
] = DeepseekSparseSWAMetadataBuilder,
|
||||
|
||||
@@ -10,9 +10,11 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.v1.attention.backends.mla.indexer as indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.config import AttentionConfig, CUDAGraphMode
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
|
||||
|
||||
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
|
||||
|
||||
|
||||
def _chunk(start: int, end: int) -> SimpleNamespace:
|
||||
"""Chunk over rows [start, end); bounds derive from the global row."""
|
||||
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
|
||||
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
"""The split covers every row once with >= 1 row per rank and declines
|
||||
below the activation floor (or on a single rank)."""
|
||||
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
|
||||
n = MIN_ROWS
|
||||
shapes = [
|
||||
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
|
||||
# uneven requests at different context depths
|
||||
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
torch.tensor(query_lens, dtype=torch.int32),
|
||||
ratio,
|
||||
tp_size,
|
||||
n,
|
||||
)
|
||||
assert sizes is not None and len(sizes) == tp_size
|
||||
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
|
||||
lens = torch.tensor([n - 1], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
|
||||
|
||||
|
||||
def test_row_shard_threshold_is_total_not_per_rank() -> None:
|
||||
"""The activation floor counts TOTAL rows regardless of tp_size: a
|
||||
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
|
||||
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
|
||||
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
|
||||
for tp_size in (2, 4, 8):
|
||||
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
|
||||
assert sizes is not None
|
||||
|
||||
|
||||
def test_tp2_old_per_rank_window_stays_replicated() -> None:
|
||||
"""32K-64K total rows engaged on TP2 under the old per-rank floor
|
||||
(16K rows/rank); the total floor now keeps them replicated — the
|
||||
all_gatherv latency term is largest at small tp."""
|
||||
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
|
||||
|
||||
|
||||
def test_row_shard_floor_is_at_least_tp_size() -> None:
|
||||
"""A floor below tp_size still leaves every rank at least one row."""
|
||||
lens = torch.tensor([8], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
|
||||
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
|
||||
|
||||
|
||||
def test_row_sharding_gate_rejects_unsupported_configurations(
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Equivalence tests for the fused mamba "align" block-table gather.
|
||||
|
||||
`mamba_get_block_table_tensor`'s align branch used to be five PyTorch
|
||||
statements that expanded to seven kernel launches over a `(num_reqs,)` tensor.
|
||||
It is now one Triton kernel. These tests pin the fused kernel to the exact
|
||||
semantics of the expression it replaced, including cases a single-request
|
||||
decode never exercises but other callers can hit.
|
||||
|
||||
`block_table` is deliberately tested non-contiguous: the original
|
||||
`torch.gather(block_table, 1, ...)` honours any layout, and this is a shared
|
||||
utility (`gdn_attn`, `mamba_attn`, `short_conv_attn` and `linear_attn` all call
|
||||
it), so a column stride of 1 must not be assumed.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
WIDTH = 64
|
||||
|
||||
|
||||
def _spec(block_size: int, num_speculative_blocks: int) -> MambaSpec:
|
||||
# Only block_size and num_speculative_blocks are read by the align branch;
|
||||
# shapes/dtypes just have to be well formed for the frozen dataclass.
|
||||
return MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,),),
|
||||
dtypes=(torch.float32,),
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
|
||||
def _reference(block_table, seq_lens, block_size, num_spec_blocks):
|
||||
"""The align branch exactly as it was before the fusion."""
|
||||
start_indices = (seq_lens - 1) // block_size
|
||||
start_indices = start_indices.clamp(min=0)
|
||||
offsets = torch.arange(
|
||||
1 + num_spec_blocks, device=block_table.device, dtype=torch.int32
|
||||
)
|
||||
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
|
||||
return torch.gather(block_table, 1, indices_to_gather)
|
||||
|
||||
|
||||
def _make(nreq, block_size, nspec, device, noncontig=False):
|
||||
torch.manual_seed(0)
|
||||
width = WIDTH * 2 if noncontig else WIDTH
|
||||
bt = torch.randint(0, 5000, (nreq, width), dtype=torch.int32, device=device)
|
||||
if noncontig:
|
||||
# Every other column: stride(1) == 2, which a kernel assuming a unit
|
||||
# column stride would read straight through, silently returning the
|
||||
# wrong blocks.
|
||||
bt = bt[:, ::2]
|
||||
assert not bt.is_contiguous()
|
||||
# Keep start + nspec inside the row, which the gather already requires.
|
||||
hi = max(1, (WIDTH - 1 - nspec) * block_size)
|
||||
sl = torch.randint(0, hi, (nreq,), dtype=torch.int32, device=device)
|
||||
return bt, sl
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
@pytest.mark.parametrize("nreq", [1, 2, 8, 64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 256])
|
||||
@pytest.mark.parametrize("nspec", [0, 1, 2, 3, 4, 7])
|
||||
def test_matches_reference(nreq, block_size, nspec):
|
||||
bt, sl = _make(nreq, block_size, nspec, "cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
|
||||
want = _reference(bt, sl, block_size, nspec)
|
||||
assert got.shape == want.shape
|
||||
assert got.dtype == want.dtype
|
||||
torch.testing.assert_close(got, want, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
@pytest.mark.parametrize("nspec", [0, 3, 7])
|
||||
def test_non_contiguous_block_table(nspec):
|
||||
"""A column stride != 1 must be honoured, not assumed away."""
|
||||
bt, sl = _make(8, 64, nspec, "cuda", noncontig=True)
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, nspec), "align")
|
||||
want = _reference(bt, sl, 64, nspec)
|
||||
torch.testing.assert_close(got, want, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_zero_seq_len_clamps_to_first_row():
|
||||
"""CUDA-graph padding rows carry seq_len 0; the old code clamped to row 0."""
|
||||
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.zeros(4, dtype=torch.int32, device="cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
|
||||
torch.testing.assert_close(got, _reference(bt, sl, 64, 3), atol=0, rtol=0)
|
||||
# start clamps to 0, so this is just the first 1 + nspec columns.
|
||||
torch.testing.assert_close(got, bt[:, :4], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_zero_requests_returns_empty():
|
||||
"""An empty batch must not launch a (0,) grid."""
|
||||
bt = torch.zeros((0, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.zeros(0, dtype=torch.int32, device="cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
|
||||
assert got.shape == (0, 4)
|
||||
assert got.dtype == bt.dtype
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_none_mode_returns_the_input_unchanged():
|
||||
"""`MambaCacheMode` is Literal["align", "none"]; only "none" passes through.
|
||||
|
||||
An earlier version of this test also parametrised a "mode" of "all", which
|
||||
is not a member of that Literal. Anything that is not "none" takes the
|
||||
align branch and returns a NEW tensor, so asserting identity for it failed
|
||||
-- correctly. The passthrough contract applies to "none" alone.
|
||||
"""
|
||||
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.randint(0, 1000, (4,), dtype=torch.int32, device="cuda")
|
||||
assert mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "none") is bt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CPU tensors. Every test above runs on "cuda", which is exactly how the
|
||||
# regression reached review: `mamba_get_block_table_tensor` is a SHARED
|
||||
# utility and its callers legitimately pass CPU tensors -- the GDN and
|
||||
# linear-attention metadata builders are unit-tested with
|
||||
# `DEVICE = torch.device("cpu")`. A Triton kernel cannot run on those at all,
|
||||
# while the torch expression it replaced could. These run without a GPU.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nspec", [0, 1, 3])
|
||||
@pytest.mark.parametrize("block_size", [1, 16, 64])
|
||||
def test_cpu_tensors_match_the_reference(nspec, block_size):
|
||||
"""The align branch must work on CPU, not just on the accelerator."""
|
||||
width = 2 + nspec + 8
|
||||
bt = torch.arange(5 * width, dtype=torch.int32).reshape(5, width)
|
||||
sl = torch.tensor([0, 1, block_size, block_size * 3 + 1, 7], dtype=torch.int32)
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
|
||||
want = _reference(bt, sl, block_size, nspec)
|
||||
assert torch.equal(got, want), f"cpu mismatch: {got.tolist()} != {want.tolist()}"
|
||||
assert got.device == bt.device
|
||||
assert got.dtype == bt.dtype
|
||||
|
||||
|
||||
def test_cpu_none_mode_passes_through():
|
||||
bt = torch.arange(12, dtype=torch.int32).reshape(3, 4)
|
||||
sl = torch.tensor([1, 2, 3], dtype=torch.int32)
|
||||
assert mamba_get_block_table_tensor(bt, sl, _spec(16, 3), "none") is bt
|
||||
@@ -171,6 +171,11 @@ def test_paged_prefill_indices_match_gathered_prefill(
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
|
||||
|
||||
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
|
||||
builder_cls = (
|
||||
rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder
|
||||
if model == "deepseek_v41"
|
||||
else rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder
|
||||
)
|
||||
device = torch.device("cuda")
|
||||
gen = torch.Generator().manual_seed(compress_ratio)
|
||||
window, width, block_size, max_model_len = 16, 32, 16, 128
|
||||
@@ -233,9 +238,7 @@ def test_paged_prefill_indices_match_gathered_prefill(
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled
|
||||
)
|
||||
return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["swa"], vllm_config, device
|
||||
).build(0, common)
|
||||
return builder_cls(swa_spec, ["swa"], vllm_config, device).build(0, common)
|
||||
|
||||
assert build(enabled=False).prefill_swa_ragged_indices is None
|
||||
md = build(enabled=True)
|
||||
|
||||
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
@@ -44,6 +46,48 @@ def cleared_attention_selector_cache():
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
def test_deepseek_rocm_backend_registry_paths_are_model_specific():
|
||||
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV4.get_path() == (
|
||||
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
|
||||
)
|
||||
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV41.get_path() == (
|
||||
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name, expect_warning",
|
||||
[
|
||||
("ROCM_FLASHMLA_SPARSE_DSV4", True),
|
||||
("ROCM_FLASHMLA_SPARSE_DSV41", False),
|
||||
],
|
||||
ids=["legacy", "canonical"],
|
||||
)
|
||||
def test_deepseek_v41_rocm_accepts_legacy_backend_alias(backend_name, expect_warning):
|
||||
from vllm.models.deepseek_v41.amd import model as dsv41_model
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV41ROCMAiterMLAAttention,
|
||||
DeepseekV41ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=AttentionConfig(backend=backend_name),
|
||||
parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
with patch.object(dsv41_model.logger, "warning_once") as warning_once:
|
||||
selected = dsv41_model._select_dsv4_attn_cls(config)
|
||||
|
||||
assert selected is DeepseekV41ROCMAiterMLAAttention
|
||||
if expect_warning:
|
||||
warning_once.assert_called_once()
|
||||
else:
|
||||
warning_once.assert_not_called()
|
||||
assert (
|
||||
DeepseekV41ROCMAiterMLASparseBackend.get_name() == "ROCM_FLASHMLA_SPARSE_DSV41"
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||
RocmAiterUnifiedAttentionBackend,
|
||||
|
||||
@@ -180,7 +180,7 @@ def create_standard_kv_cache_spec(
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||
model_name: str = "Qwen/Qwen3-0.6B",
|
||||
tensor_parallel_size: int = 1,
|
||||
max_model_len: int = 1024,
|
||||
dtype: ModelDType | torch.dtype = "auto",
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
make_block_hash_with_group_id,
|
||||
null_head_len,
|
||||
tensor_data,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
@@ -2156,3 +2157,20 @@ def test_wrapped_mamba_group_requires_block_zeroing():
|
||||
)
|
||||
|
||||
assert config.needs_kv_cache_zeroing
|
||||
|
||||
|
||||
def test_null_head_len():
|
||||
null = KVCacheBlock(0, is_null=True)
|
||||
real = [KVCacheBlock(i) for i in range(1, 4)]
|
||||
tables = [
|
||||
[],
|
||||
real,
|
||||
[null] * 5,
|
||||
[null] * 5 + real,
|
||||
[null] * 2 + real[:1] + [null] * 3 + real[1:], # not one contiguous head
|
||||
]
|
||||
for table in tables:
|
||||
expected = next((i for i, b in enumerate(table) if not b.is_null), len(table))
|
||||
assert null_head_len(table) == expected
|
||||
blocks = KVCacheBlocks(([null] * 4 + real, [null] + real[:1] + [null]))
|
||||
assert blocks.get_unhashed_block_ids_all_groups() == [[1, 2, 3], [1]]
|
||||
|
||||
@@ -558,6 +558,249 @@ def test_schedule_prefills_gating(has_running: bool):
|
||||
assert any(r.req_id == "new0" for r in output.scheduled_new_reqs)
|
||||
|
||||
|
||||
def _decode_one(scheduler, output, req_ids: list[str]) -> None:
|
||||
"""Feed back one sampled token per request, so they keep decoding."""
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
|
||||
sampled_token_ids=[[0] for _ in req_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _start_decoding(scheduler, req_id: str = "run0"):
|
||||
"""Prefill one request and take it into the decode state."""
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=8, max_tokens=100, req_ids=[req_id]
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
output = scheduler.schedule()
|
||||
assert req_id in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, [req_id])
|
||||
assert len(scheduler.running) == 1
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 4])
|
||||
def test_local_prefill_interval_holds_prefill_between_bursts(interval: int):
|
||||
"""A single engine applies `prefill_schedule_interval` in the scheduler,
|
||||
timed from the last prefill-carrying step: a waiting prefill is held for the
|
||||
remainder of the interval while running requests keep decoding, then
|
||||
admitted once the interval has elapsed.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
# This prefill starts the interval.
|
||||
_start_decoding(scheduler)
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
# Steps inside the interval decode alone; the waiting prefill is held.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "new0" not in output.num_scheduled_tokens
|
||||
assert "run0" in output.num_scheduled_tokens
|
||||
assert new_req.status == RequestStatus.WAITING
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
# The interval has now elapsed, so prefill is admitted again.
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_empty_schedule_does_not_advance_the_prefill_cadence():
|
||||
"""An empty schedule() must not consume interval budget.
|
||||
|
||||
`current_step` counts every schedule() call. With async scheduling,
|
||||
step_with_batch_queue() calls schedule() again whenever the batch queue has
|
||||
room and has_requests() is true -- so a call lands while the previous batch
|
||||
is still in flight and nothing new can be scheduled. An async KV connector
|
||||
makes this common, since requests sit in lookup or WAITING_FOR_REMOTE_KVS.
|
||||
Timing the cadence on current_step lets those empty calls expire the hold
|
||||
early. Reported on #54627 against the LMCache MP connector.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# One real decode step, left in flight (no update_from_output yet).
|
||||
output = scheduler.schedule()
|
||||
assert output.total_num_scheduled_tokens > 0
|
||||
executed_before = scheduler.num_executed_steps
|
||||
step_before = scheduler.current_step
|
||||
|
||||
# Re-entering schedule() while that batch is still unaccounted for
|
||||
# schedules nothing -- this is the async-scheduling shape.
|
||||
for _ in range(3):
|
||||
empty = scheduler.schedule()
|
||||
assert empty.total_num_scheduled_tokens == 0
|
||||
|
||||
assert scheduler.current_step == step_before + 3, "current_step should advance"
|
||||
assert scheduler.num_executed_steps == executed_before, (
|
||||
"an empty schedule() must not count as an executed step"
|
||||
)
|
||||
|
||||
|
||||
def test_prefill_cadence_counts_executed_steps_not_schedule_calls():
|
||||
"""The hold spans `interval` EXECUTED steps, however many empty schedule()
|
||||
calls are interleaved.
|
||||
|
||||
Counting schedule() calls instead lets each empty call eat interval budget,
|
||||
so with an async KV connector prefill lands on nearly every step and the
|
||||
feature stops doing anything.
|
||||
"""
|
||||
interval = 4
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
_start_decoding(scheduler)
|
||||
prefill_at = scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
empty_calls = 0
|
||||
admitted_at = None
|
||||
for _ in range(50):
|
||||
output = scheduler.schedule()
|
||||
if "new0" in output.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
if output.total_num_scheduled_tokens == 0:
|
||||
empty_calls += 1
|
||||
continue
|
||||
# A real decode step. Leave it in flight and re-enter schedule() twice,
|
||||
# as async scheduling does while the batch queue has room.
|
||||
for _ in range(2):
|
||||
extra = scheduler.schedule()
|
||||
if "new0" in extra.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
empty_calls += 1
|
||||
if admitted_at is not None:
|
||||
break
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
assert admitted_at is not None, "prefill was never admitted"
|
||||
assert empty_calls > 0, "test never exercised an empty schedule() call"
|
||||
# The hold is measured in executed steps. Counting schedule() calls would
|
||||
# admit after far fewer, because the empty re-entries above would count.
|
||||
assert admitted_at - prefill_at == interval, (
|
||||
f"hold lasted {admitted_at - prefill_at} executed steps, expected "
|
||||
f"{interval}; empty schedule() calls were counted against the cadence"
|
||||
)
|
||||
assert scheduler.current_step > scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_local_prefill_interval_defaults_to_no_gating():
|
||||
"""The default interval of 1 leaves admission exactly as it was: a waiting
|
||||
prefill is admitted on the very next step.
|
||||
"""
|
||||
scheduler = create_scheduler()
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_left_to_engine_under_dp():
|
||||
"""Under data parallelism the cadence must stay aligned across ranks, so the
|
||||
DP engine core drives it and the scheduler must not gate on its own.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4, data_parallel_size=2)
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_never_idles_the_model():
|
||||
"""With nothing running to decode in prefill's place, the interval must not
|
||||
hold prefill back, or the step would be wasted entirely.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=8)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# Finish the only running request, leaving nothing to decode.
|
||||
scheduler.finish_requests("run0", RequestStatus.FINISHED_ABORTED)
|
||||
assert not scheduler.running
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 3])
|
||||
def test_local_prefill_interval_defers_inflight_prefill_chunk(interval: int):
|
||||
"""The single-engine interval gates prefill WORK, not just new admissions.
|
||||
|
||||
A request that is already partially prefilled is held for the remainder of
|
||||
the interval too, exactly as `throttle_prefills` does for DP. That is the
|
||||
point of the cadence -- a held step must carry no prefill at all, or decode
|
||||
does not get the uninterrupted stretch the interval exists to give it.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=50,
|
||||
enable_chunked_prefill=True,
|
||||
prefill_schedule_interval=interval,
|
||||
)
|
||||
|
||||
# A short request that finishes prefill in one step -> a running decode.
|
||||
(decode_req,) = create_requests(num_requests=1, num_tokens=4, req_ids=["dec0"])
|
||||
scheduler.add_request(decode_req)
|
||||
output = scheduler.schedule()
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
assert decode_req in scheduler.running and not decode_req.is_prefill_chunk
|
||||
|
||||
# A long request (80 tokens, budget 50) -> prefilled in chunks. That first
|
||||
# prefill closed the interval, so walk to the next step that opens it.
|
||||
(chunk_req,) = create_requests(num_requests=1, num_tokens=80, req_ids=["chk0"])
|
||||
scheduler.add_request(chunk_req)
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens["chk0"] > 0
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=["dec0", "chk0"],
|
||||
req_id_to_index={"dec0": 0, "chk0": 1},
|
||||
sampled_token_ids=[[0], []], # no token sampled for partial prefill
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
assert chunk_req.is_prefill_chunk # still mid-prefill, in running
|
||||
|
||||
# Held steps: the in-flight chunk is deferred along with new admissions,
|
||||
# while the decode keeps running.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
assert "dec0" in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
# Once the interval has elapsed the chunk resumes.
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def _setup_remote_kv_resume(num_prompt_tokens: int, matched_tokens: int):
|
||||
"""Drive a remote-KV request `r2` to the resume point (async load complete)
|
||||
while another request `r1` is already decoding, so the step is throttle-
|
||||
@@ -1916,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
|
||||
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
|
||||
def test_block_drafters_cap_requests_instead_of_draft_slots(
|
||||
tmp_path, method, max_num_reqs
|
||||
):
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
base = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
max_num_batched_tokens=20,
|
||||
num_speculative_tokens=4,
|
||||
parallel_drafting=True,
|
||||
skip_tokenizer_init=True,
|
||||
)
|
||||
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
|
||||
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
|
||||
vllm_config = base.vllm_config
|
||||
object.__setattr__(vllm_config.speculative_config, "method", method)
|
||||
object.__setattr__(
|
||||
vllm_config.speculative_config,
|
||||
"draft_model_config",
|
||||
SimpleNamespace(hf_config=SimpleNamespace()),
|
||||
)
|
||||
vllm_config._set_max_num_scheduled_tokens()
|
||||
scheduler = Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=base.kv_cache_config,
|
||||
structured_output_manager=base.structured_output_manager,
|
||||
block_size=base.block_size,
|
||||
)
|
||||
for request in create_requests(num_requests=8, num_tokens=2):
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
|
||||
# charged: each admitted prompt is scheduled in full.
|
||||
assert scheduler.schedule().num_scheduled_tokens == {
|
||||
str(i): 2 for i in range(max_num_reqs)
|
||||
}
|
||||
|
||||
|
||||
# Note - these test cases mirror some of those in test_rejection_sampler.py
|
||||
@pytest.mark.parametrize(
|
||||
"spec_tokens,output_tokens,expected,expected_per_req",
|
||||
|
||||
@@ -1013,6 +1013,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
|
||||
)
|
||||
total_computed = num_tokens
|
||||
|
||||
|
||||
def test_mamba_honors_drop_eagle_block():
|
||||
"""MambaManager must drop the EAGLE lookahead block, like FullAttentionManager.
|
||||
|
||||
@@ -1067,7 +1068,7 @@ def test_mamba_honors_drop_eagle_block():
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=block_size,
|
||||
)[0]
|
||||
)[0][0]
|
||||
)
|
||||
|
||||
# Control: with no eagle drop, both groups see the whole cached prefix.
|
||||
@@ -1082,3 +1083,35 @@ def test_mamba_honors_drop_eagle_block():
|
||||
f"MambaManager ignored drop_eagle_block: full-attention dropped the "
|
||||
f"lookahead block (hit={full_hit}), mamba kept it (hit={mamba_hit})"
|
||||
)
|
||||
|
||||
|
||||
def test_pop_blocks_for_free_trims_null_head():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=4,
|
||||
)
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
)
|
||||
manager = get_sliding_window_manager(spec, block_pool)
|
||||
null = block_pool.null_block
|
||||
real = block_pool.get_new_blocks(2)
|
||||
manager.req_to_blocks["swa"] = [null] * 3 + real
|
||||
manager.req_to_blocks["dense"] = dense = block_pool.get_new_blocks(2)
|
||||
|
||||
assert manager.pop_blocks_for_free("swa") == real
|
||||
assert manager.pop_blocks_for_free("dense") is dense # no null head, no copy
|
||||
assert "swa" not in manager.req_to_blocks
|
||||
assert all(b.ref_cnt == 1 for b in real + dense)
|
||||
|
||||
# The null block is never pooled, so touch / free_blocks leave it alone.
|
||||
null_ref_cnt = null.ref_cnt
|
||||
num_free_blocks = block_pool.get_num_free_blocks()
|
||||
block_pool.touch([null])
|
||||
block_pool.free_blocks([null])
|
||||
assert null.ref_cnt == null_ref_cnt
|
||||
assert block_pool.get_num_free_blocks() == num_free_blocks
|
||||
|
||||
@@ -85,6 +85,7 @@ def create_scheduler(
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
runner: RunnerOption = "auto",
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
prefill_schedule_interval: int = 1,
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
scheduler_cls: type[Scheduler] | None = None,
|
||||
@@ -134,6 +135,7 @@ def create_scheduler(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
async_scheduling=async_scheduling,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
prefill_schedule_interval=prefill_schedule_interval,
|
||||
# Ensure admission/preemption mechanics are deterministic
|
||||
watermark=0.0,
|
||||
policy=scheduling_policy,
|
||||
|
||||
@@ -160,6 +160,8 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
Skips the parts of __init__ that require a real VllmConfig / model
|
||||
by patching the attributes directly after construction.
|
||||
"""
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
|
||||
|
||||
mgr = object.__new__(EncoderCudaGraphManager)
|
||||
mgr.token_budgets = sorted(budgets)
|
||||
mgr.path_token_budgets = {"default": mgr.token_budgets}
|
||||
@@ -1044,6 +1046,278 @@ class TestGetInputModality:
|
||||
assert model.get_input_modality(mm_kwargs) == "video"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU tests — DeepSeek-VL2 batched-tile encoder CUDA graph
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# DeepSeek-VL2 mock constants (kept small for fast capture)
|
||||
_DSV2_H = 2 # projector output side per tile (h = w)
|
||||
_DSV2_DIM = 16 # hidden size
|
||||
_DSV2_IMG_SIZE = 4 # each tile is 4×4 pixels
|
||||
_DSV2_BUDGETS = [16, 64]
|
||||
_DSV2_MAX_BATCH = 4
|
||||
|
||||
|
||||
def _dsv2_output_tokens(tw: int, th: int) -> int:
|
||||
"""Expected output tokens for an image with (tw, th) local tiles."""
|
||||
h = w = _DSV2_H
|
||||
return h * (w + 1) + th * h * (tw * w + 1) + 1
|
||||
|
||||
|
||||
def _make_deepseek_vl2_mm_kwargs(
|
||||
spatial_crops: list[list[int]],
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> dict[str, Any]:
|
||||
"""spatial_crops: [[tw, th], ...] per image."""
|
||||
total_tiles = sum(1 + int(tw) * int(th) for tw, th in spatial_crops)
|
||||
pixel_values = torch.randn(
|
||||
total_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
|
||||
)
|
||||
return {
|
||||
"pixel_values": pixel_values,
|
||||
"images_spatial_crop": torch.tensor(
|
||||
spatial_crops, dtype=torch.long, device=device
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class MockDeepseekVL2Model(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
"""Mock of DeepseekVLV2ForCausalLM for CUDA graph tests.
|
||||
|
||||
Simulates the batched-tile ViT pattern: pixel_values is
|
||||
[N_tiles, 3, H, W], output is assembled with newlines and
|
||||
view-separator in postprocess_encoder_output.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
in_features = 3 * _DSV2_IMG_SIZE * _DSV2_IMG_SIZE
|
||||
self.proj = torch.nn.Linear(in_features, hw * _DSV2_DIM)
|
||||
self.image_newline = torch.nn.Parameter(torch.randn(_DSV2_DIM))
|
||||
self.view_seperator = torch.nn.Parameter(torch.randn(_DSV2_DIM))
|
||||
self.global_view_pos = "head"
|
||||
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
return 1
|
||||
|
||||
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=_DSV2_DIM,
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(self, vllm_config) -> tuple[int, int]:
|
||||
return (_dsv2_output_tokens(1, 1), 128)
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[EncoderItemSpec]:
|
||||
h = w = _DSV2_H
|
||||
specs = []
|
||||
for row in mm_kwargs["images_spatial_crop"].tolist():
|
||||
tw, th = int(row[0]), int(row[1])
|
||||
if tw == 0 or th == 0:
|
||||
break
|
||||
specs.append(
|
||||
EncoderItemSpec(
|
||||
input_size=1 + tw * th,
|
||||
output_tokens=h * (w + 1) + th * h * (tw * w + 1) + 1,
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
) -> dict[str, Any]:
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
images_spatial_crop = mm_kwargs["images_spatial_crop"]
|
||||
|
||||
if not indices:
|
||||
return {
|
||||
"pixel_values": pixel_values[:0],
|
||||
"images_spatial_crop": images_spatial_crop[:0],
|
||||
}
|
||||
|
||||
tiles_per_image = [
|
||||
1 + int(row[0]) * int(row[1]) for row in images_spatial_crop.tolist()
|
||||
]
|
||||
cum = [0]
|
||||
for t in tiles_per_image:
|
||||
cum.append(cum[-1] + t)
|
||||
|
||||
selected_pv = torch.cat([pixel_values[cum[i] : cum[i + 1]] for i in indices])
|
||||
return {
|
||||
"pixel_values": selected_pv,
|
||||
"images_spatial_crop": images_spatial_crop[list(indices)],
|
||||
}
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
max_tiles = (token_budget + hw - 1) // hw
|
||||
dummy = torch.randn(
|
||||
max_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
|
||||
)
|
||||
return EncoderCudaGraphCaptureInputs(values={"pixel_values": dummy})
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphReplayBuffers:
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={"pixel_values": mm_kwargs["pixel_values"]}
|
||||
)
|
||||
|
||||
def _tile_forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
"""Project tiles: [N, 3, H, W] → [N, hw, DIM]."""
|
||||
n = pixel_values.shape[0]
|
||||
flat = pixel_values.view(n, -1)
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
return self.proj(flat).view(n, hw, _DSV2_DIM)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, values: dict[str, torch.Tensor], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
return self._tile_forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self, mm_kwargs: dict[str, Any], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
# Manager always runs `postprocess_encoder_output` after this, whether
|
||||
# the batch went through eager or CUDA graph replay, so this must
|
||||
# return the same raw per-tile shape as `encoder_cudagraph_forward`.
|
||||
return self._tile_forward(mm_kwargs["pixel_values"])
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
assert batch_mm_kwargs is not None
|
||||
output = outputs["default"]
|
||||
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
|
||||
_, hw, n_dim = output.shape
|
||||
h = w = int(hw**0.5)
|
||||
|
||||
tile_offset = 0
|
||||
for rank, img_idx in enumerate(indices):
|
||||
tw = int(images_spatial_crop[rank][0])
|
||||
th = int(images_spatial_crop[rank][1])
|
||||
n_tiles = 1 + tw * th
|
||||
tiles = output[tile_offset : tile_offset + n_tiles]
|
||||
tile_offset += n_tiles
|
||||
|
||||
global_f = tiles[0].view(h, w, n_dim)
|
||||
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
|
||||
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
|
||||
|
||||
local_f = tiles[1:].view(th, tw, h, w, n_dim)
|
||||
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
|
||||
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
|
||||
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
|
||||
|
||||
if self.global_view_pos == "head":
|
||||
emb = torch.cat([global_f, self.view_seperator[None], local_f])
|
||||
else:
|
||||
emb = torch.cat([local_f, self.view_seperator[None], global_f])
|
||||
|
||||
if isinstance(dest, dict):
|
||||
dest[img_idx] = emb.clone() if clone else emb
|
||||
else:
|
||||
dest[rank] = emb.clone() if clone else emb
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
class TestDeepseekVL2CudaGraph:
|
||||
"""CUDA graph tests for the batched-tile encoder pattern (DeepSeek-VL2)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.device = torch.device("cuda:0")
|
||||
self.dtype = torch.float16
|
||||
self.model = MockDeepseekVL2Model().to(self.device).half()
|
||||
self.mgr = _make_manager_for_gpu(
|
||||
self.model, _DSV2_BUDGETS, _DSV2_MAX_BATCH, self.device, self.dtype
|
||||
)
|
||||
self.graph_pool = current_platform.graph_pool_handle()
|
||||
self.mgr.capture(graph_pool=self.graph_pool)
|
||||
|
||||
def test_capture_creates_one_graph_per_budget(self):
|
||||
assert set(self.mgr.budget_graphs["default"].keys()) == set(_DSV2_BUDGETS)
|
||||
|
||||
def test_execute_returns_one_tensor_per_image(self):
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [1, 1]], self.device, self.dtype
|
||||
)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert len(result) == 2
|
||||
|
||||
def test_output_tokens_1x1_tile(self):
|
||||
# global=h*(w+1)=2*3=6, local=1*2*(1*2+1)=6, sep=1 → 13
|
||||
expected = _dsv2_output_tokens(1, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[1, 1]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
|
||||
def test_output_tokens_2x1_tile(self):
|
||||
# global=6, local=1*2*(2*2+1)=10, sep=1 → 17
|
||||
expected = _dsv2_output_tokens(2, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[2, 1]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
|
||||
def test_output_tokens_multi_image(self):
|
||||
# Image 0: tw=1,th=1 → 13; Image 1: tw=2,th=1 → 17
|
||||
e0 = _dsv2_output_tokens(1, 1)
|
||||
e1 = _dsv2_output_tokens(2, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [2, 1]], self.device, self.dtype
|
||||
)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (e0, _DSV2_DIM)
|
||||
assert result[1].shape == (e1, _DSV2_DIM)
|
||||
|
||||
def test_eager_fallback_when_tokens_exceed_all_budgets(self):
|
||||
# tw=4,th=4: global=6, local=4*2*(4*2+1)=72, sep=1 → 79 > max budget 64
|
||||
tw, th = 4, 4
|
||||
expected = _dsv2_output_tokens(tw, th)
|
||||
assert expected > _DSV2_BUDGETS[-1], "test precondition: must exceed max budget"
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[tw, th]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
assert self.mgr.graph_misses >= 1
|
||||
|
||||
def test_graph_hit_counter(self):
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [1, 1]], self.device, self.dtype
|
||||
)
|
||||
self.mgr.execute(mm_kwargs)
|
||||
assert self.mgr.graph_hits == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU tests — video capture, replay, fallback, and mixed image+video
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert engine.engine_core.dp_engines_running(), (
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
"the coordinator does not report the engines as running while they are"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,17 +4,28 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
class DictStore:
|
||||
def __init__(self, values: dict[str, bytes]):
|
||||
self.values = values
|
||||
|
||||
def check(self, keys: list[str]) -> bool:
|
||||
return all(key in self.values for key in keys)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self.values[key]
|
||||
|
||||
def set(self, key: str, value: bytes) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_master_port=12345,
|
||||
_data_parallel_master_port_list=[12346, 12347],
|
||||
)
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
control_store, reconfig_store = DictStore({}), DictStore({})
|
||||
coordinator.reconfig_store_ref = reconfig_store
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
|
||||
)
|
||||
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
|
||||
|
||||
for store in (control_store, reconfig_store):
|
||||
# DictStore preserves insertion order: metadata must precede the epoch.
|
||||
assert list(store.values)[-1] == coordinator.key("current_epoch")
|
||||
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
|
||||
bootstrap
|
||||
)
|
||||
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
|
||||
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_get_reconfig_store", lambda store=store: store
|
||||
)
|
||||
assert coordinator.get_status()["requested_data_parallel_size"] == 3
|
||||
|
||||
|
||||
def test_external_elastic_ep_calculates_target_expert_redundancy():
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0)
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
|
||||
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dp_rank", range(4))
|
||||
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
|
||||
"""Validate expert capacity on every rank before waiting for an epoch."""
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_rank=dp_rank,
|
||||
elastic_ep_max_dp_size=4,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
_coord_store_port=12345,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
store = DictStore({})
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot retain all model experts"):
|
||||
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
|
||||
|
||||
assert not store.values
|
||||
assert coordinator.prepared_scale is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_elastic_ep_late_rank_observes_epoch_error():
|
||||
coordinator = ExternalElasticEPScaleCoordinator(
|
||||
cast("DPAsyncMPClient", SimpleNamespace())
|
||||
)
|
||||
epoch = "failed-epoch"
|
||||
store = DictStore(
|
||||
{
|
||||
coordinator.key("current_epoch"): epoch.encode(),
|
||||
coordinator.key(epoch, "error"): b"scale failed",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scale failed"):
|
||||
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
|
||||
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def start_rank(
|
||||
self,
|
||||
rank: int,
|
||||
dp_size: int | None = None,
|
||||
elastic_ep_scale_up_launch: bool = False,
|
||||
) -> tuple[RemoteOpenAIServer, list[str]]:
|
||||
"""Start one externally managed DP rank."""
|
||||
dp_size = self.dp_size if dp_size is None else dp_size
|
||||
server_args = self.base_server_args.copy()
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank),
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
),
|
||||
}
|
||||
if elastic_ep_scale_up_launch:
|
||||
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
|
||||
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
server_args,
|
||||
auto_port=False,
|
||||
env_dict=env_dict,
|
||||
)
|
||||
print(
|
||||
f"Server rank {rank} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
server_and_args = (server, server_args)
|
||||
self.servers.append(server_and_args)
|
||||
return server_and_args
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start all server instances for external LB mode."""
|
||||
"""Start all initial server instances for external LB mode."""
|
||||
start_errors: list[Exception] = []
|
||||
for rank in range(self.dp_size):
|
||||
# Create server args for this specific rank
|
||||
server_args = self.base_server_args.copy()
|
||||
|
||||
# Add external LB specific arguments
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(self.dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank), # Different port for each rank
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
# Use a thread to start each server to allow parallel initialization
|
||||
def start_server(r: int, sargs: list[str]):
|
||||
def start_server(r: int):
|
||||
try:
|
||||
# Start the server
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
sargs,
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
|
||||
),
|
||||
},
|
||||
)
|
||||
server.__enter__()
|
||||
print(
|
||||
f"Server rank {r} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
self.servers.append((server, sargs))
|
||||
self.start_rank(r)
|
||||
except Exception as e:
|
||||
print(f"Failed to start server rank {r}: {e}")
|
||||
raise
|
||||
start_errors.append(e)
|
||||
|
||||
thread = threading.Thread(target=start_server, args=(rank, server_args))
|
||||
thread = threading.Thread(target=start_server, args=(rank,))
|
||||
thread.start()
|
||||
|
||||
self.server_threads.append(thread)
|
||||
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
|
||||
for thread in self.server_threads:
|
||||
thread.join()
|
||||
|
||||
if start_errors:
|
||||
start_error = start_errors[0]
|
||||
raise RuntimeError("Failed to start external LB servers") from start_error
|
||||
|
||||
# Give servers additional time to fully initialize and coordinate
|
||||
time.sleep(2)
|
||||
|
||||
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
_assert_expert_parallel_config(servers)
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
|
||||
@@ -5,7 +5,13 @@ import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import (
|
||||
assert_request_outputs_match,
|
||||
compute_acceptance_len,
|
||||
get_test_prompts,
|
||||
)
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@@ -37,3 +43,107 @@ def test_mtp_correctness(
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_mtp_draft_lm_head_quantization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""A quantized draft copy of the shared lm_head only changes drafting:
|
||||
greedy outputs match stock MTP and acceptance stays within 10%."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
results = {}
|
||||
# NVFP4 needs a CUDA W4A16 kernel.
|
||||
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
|
||||
for quant in quants:
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": quant,
|
||||
},
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
outputs = runner.llm.chat(prompts, sampling_config)
|
||||
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
|
||||
|
||||
ref_outputs, ref_al = results.pop(None)
|
||||
for quant, (outputs, al) in results.items():
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
# Not bit-exact: a different acceptance length changes the target's
|
||||
# verify batch shapes, which can flip a near-tied greedy token.
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context=f"draft_lm_head_quantization={quant}",
|
||||
)
|
||||
assert al >= 0.9 * ref_al, (
|
||||
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
|
||||
)
|
||||
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""The quantized draft head is derived from the target head: it must come
|
||||
back intact after a level 1 sleep and be re-derived after a target weight
|
||||
reload, so outputs and acceptance match the run before."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": "nvfp4",
|
||||
},
|
||||
enable_sleep_mode=True,
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
|
||||
def run():
|
||||
before = llm.get_metrics()
|
||||
outputs = llm.chat(prompts, sampling_config)
|
||||
return outputs, compute_acceptance_len(llm.get_metrics(), before)
|
||||
|
||||
ref_outputs, ref_al = run()
|
||||
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up()
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after level 1 sleep",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
llm.collective_rpc("reload_weights")
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after weight reload",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
|
||||
False,
|
||||
),
|
||||
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
|
||||
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
|
||||
],
|
||||
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
|
||||
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
|
||||
) -> None:
|
||||
"""The DP master node holds a coordination store for both engine backends,
|
||||
so engines there pick world-group ports at bind time. Every other
|
||||
deployment keeps the pre-allocated ports and must not hold one."""
|
||||
so engines there pick world-group ports at bind time. Elastic EP also
|
||||
creates its coordination store before startup. The remaining cases
|
||||
below keep the pre-allocated ports and must not hold a store."""
|
||||
|
||||
class FakeManager:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
|
||||
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def getsockopt_string(self, _option):
|
||||
return "inproc://dummy"
|
||||
|
||||
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
|
||||
monkeypatch.setattr(
|
||||
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
|
||||
@@ -210,6 +213,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
client = object.__new__(DPLBAsyncMPClient)
|
||||
client.client_count = client_count
|
||||
client.reqs_in_flight = {}
|
||||
client.streaming_reqs = set()
|
||||
client.engine_inflight = Counter()
|
||||
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
|
||||
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
|
||||
@@ -407,11 +411,47 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
@pytest.mark.parametrize("final_sent", [False, True])
|
||||
def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
|
||||
"""A streaming-input chunk in flight when its session finished may
|
||||
re-create it on any engine, so an open session's abort reaches every
|
||||
engine. Other aborts go only to the routed engine."""
|
||||
client = _make_dplb_client(num_engines=3)
|
||||
client.resources = SimpleNamespace(engine_dead=False)
|
||||
sent: dict[bytes, list[str]] = {}
|
||||
|
||||
async def abort_requests(request_ids, engine):
|
||||
sent[engine] = request_ids
|
||||
|
||||
client._abort_requests = abort_requests
|
||||
|
||||
chunk = make_request(SamplingParams(max_tokens=1))
|
||||
chunk.resumable = True
|
||||
client.get_core_engine_for_request(chunk)
|
||||
if final_sent:
|
||||
chunk.resumable = False
|
||||
client.get_core_engine_for_request(chunk)
|
||||
# The engine finished the session, dropping its routing entry.
|
||||
outputs = EngineCoreOutputs(finished_requests={chunk.request_id})
|
||||
asyncio.run(DPLBAsyncMPClient.process_engine_outputs(client, outputs))
|
||||
client.reqs_in_flight["live"] = client.core_engines[1]
|
||||
|
||||
asyncio.run(client.abort_requests_async([chunk.request_id, "live"]))
|
||||
|
||||
if final_sent:
|
||||
expected = {client.core_engines[1]: ["live"]}
|
||||
else:
|
||||
expected = {engine: [chunk.request_id] for engine in client.core_engines}
|
||||
expected[client.core_engines[1]].append("live")
|
||||
assert sent == expected
|
||||
assert not client.streaming_reqs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
@@ -419,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
client.coordinator_input_address = None
|
||||
client.coordinator_output_address = None
|
||||
|
||||
payload = msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
@@ -441,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
coord_store_port=1234,
|
||||
coordinator_input_address="tcp://127.0.0.1:1235",
|
||||
coordinator_output_address="tcp://127.0.0.1:1236",
|
||||
)
|
||||
)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
@@ -461,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
assert client.vllm_config.parallel_config._coord_store_port == 1234
|
||||
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
|
||||
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
@@ -471,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
|
||||
@@ -1541,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
|
||||
OutputProcessor,
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
StreamingInputOutputCollector,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.loggers import PrometheusStatLogger
|
||||
@@ -1642,6 +1643,137 @@ async def test_cumulative_output_collector_n():
|
||||
assert third[0].text == "c"
|
||||
|
||||
|
||||
def _make_boundary_output(
|
||||
token_id: int, finish_reason: str | None, finished: bool = False
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="my-request-id",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="a",
|
||||
token_ids=[token_id],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_input_collector_preserves_chunk_boundary():
|
||||
"""A pending chunk-boundary marker (finish_reason on a non-finished
|
||||
output) must not be erased by merging the next chunk's output into it."""
|
||||
collector = StreamingInputOutputCollector(
|
||||
RequestOutputKind.DELTA, request_id="my-request-id-int"
|
||||
)
|
||||
|
||||
collector.put(_make_boundary_output(0, "stop"))
|
||||
collector.put(_make_boundary_output(1, None))
|
||||
collector.put(_make_boundary_output(2, None))
|
||||
|
||||
boundary = await collector.get()
|
||||
assert boundary.outputs[0].finish_reason == "stop"
|
||||
assert boundary.outputs[0].token_ids == [0]
|
||||
|
||||
following = await collector.get()
|
||||
assert following.outputs[0].finish_reason is None
|
||||
assert following.outputs[0].token_ids == [1, 2]
|
||||
|
||||
assert collector.output is None
|
||||
assert not collector._backlog
|
||||
assert not collector.ready.is_set()
|
||||
|
||||
|
||||
def _make_streaming_input_request(
|
||||
request_id: str, prompt_token_ids: list[int]
|
||||
) -> EngineCoreRequest:
|
||||
"""One chunk of a streaming-input (resumable) request."""
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(
|
||||
detokenize=False, output_kind=RequestOutputKind.DELTA
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
resumable=True,
|
||||
external_req_id=f"{request_id}-ext",
|
||||
)
|
||||
|
||||
|
||||
def _add_streaming_request_with_queued_chunk(
|
||||
processor: OutputProcessor, request_id: str
|
||||
) -> RequestOutputCollector:
|
||||
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
|
||||
processor.add_request(
|
||||
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
|
||||
)
|
||||
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
|
||||
return queue
|
||||
|
||||
|
||||
def test_streaming_input_stop_is_chunk_boundary():
|
||||
"""STOP is a per-chunk boundary: finished is forced False, the queued
|
||||
chunk is applied, and the request state stays alive."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[4],
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.reqs_to_abort == []
|
||||
assert processor.get_num_unfinished_requests() == 1
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is False
|
||||
assert output.outputs[0].finish_reason == "stop"
|
||||
req_state = processor.request_states["stream-req"]
|
||||
assert req_state.prompt_token_ids is not None
|
||||
assert req_state.prompt_token_ids[-2:] == [7, 8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
|
||||
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
|
||||
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
|
||||
the state is freed, and an abort is routed back to the engine core."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[],
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processor.get_num_unfinished_requests() == 0
|
||||
assert processed.reqs_to_abort == ["stream-req"]
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is True
|
||||
assert output.outputs[0].finish_reason == str(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner", ["generate", "pooling"])
|
||||
@pytest.mark.parametrize("abort_by", ["internal", "external"])
|
||||
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
|
||||
|
||||
@@ -281,6 +281,7 @@ class _FinishedProcess:
|
||||
|
||||
|
||||
def test_wait_for_engine_startup_reports_watched_process_exit():
|
||||
"""Update the startup test argument so it reaches frontend failure handling."""
|
||||
ctx = zmq.Context()
|
||||
handshake_socket = ctx.socket(zmq.ROUTER)
|
||||
recv, send = connection.Pipe(duplex=False)
|
||||
|
||||
@@ -190,11 +190,13 @@ def _apply_ft(server, instruction: str, params: dict | None = None) -> dict:
|
||||
|
||||
def _kill_worker_process(server) -> None:
|
||||
"""SIGKILL only the worker proc, leaving EngineCore and API server alive."""
|
||||
workers = [
|
||||
p
|
||||
for p in psutil.Process(server.proc.pid).children(recursive=True)
|
||||
if "Worker" in " ".join(p.cmdline())
|
||||
]
|
||||
workers = []
|
||||
for p in psutil.Process(server.proc.pid).children(recursive=True):
|
||||
# Skip descendants that already exited: cmdline() raises ZombieProcess
|
||||
# if unreaped, NoSuchProcess if reaped. A dead worker still fails below.
|
||||
with contextlib.suppress(psutil.ZombieProcess, psutil.NoSuchProcess):
|
||||
if "Worker" in " ".join(p.cmdline()):
|
||||
workers.append(p)
|
||||
assert len(workers) == 1, f"expected 1 worker proc, found: {workers}"
|
||||
workers[0].kill()
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
@@ -1551,6 +1552,95 @@ class TestNixlHandshake:
|
||||
with pytest.raises(AssertionError):
|
||||
worker.add_remote_agent(bad_meta, remote_tp_size=2)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_quant_mode", "local_tp", "remote_tp", "num_kv_heads", "rejected"),
|
||||
[
|
||||
(KVQuantMode.NVFP4, 2, 1, 8, True),
|
||||
(KVQuantMode.NVFP4, 1, 2, 8, True),
|
||||
(KVQuantMode.NVFP4, 2, 2, 8, False),
|
||||
# Both sides hold one replicated head per rank: no head split.
|
||||
(KVQuantMode.NVFP4, 16, 8, 8, False),
|
||||
(KVQuantMode.NONE, 2, 1, 8, False),
|
||||
],
|
||||
)
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_handshake_rejects_nvfp4_kv_head_split(
|
||||
self,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
kv_quant_mode,
|
||||
local_tp,
|
||||
remote_tp,
|
||||
num_kv_heads,
|
||||
rejected,
|
||||
):
|
||||
"""NVFP4 pages keep all heads' data before all heads' scales, so
|
||||
heterogeneous-TP head splitting would silently corrupt the KV.
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=max(1, num_kv_heads // local_tp),
|
||||
head_size=16,
|
||||
dtype=torch.uint8,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
|
||||
return_value=local_tp,
|
||||
):
|
||||
worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
"engine",
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
worker.transfer_topo.total_num_kv_heads = num_kv_heads
|
||||
worker.transfer_topo.local_physical_heads = max(1, num_kv_heads // local_tp)
|
||||
|
||||
local_len = 4096 * worker.block_size
|
||||
remote_len = (
|
||||
local_len
|
||||
* max(1, num_kv_heads // remote_tp)
|
||||
// worker.transfer_topo.local_physical_heads
|
||||
)
|
||||
worker.block_len_per_layer = [local_len]
|
||||
worker.num_blocks = 1
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
meta = NixlAgentMetadata(
|
||||
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
|
||||
kv_caches_base_addr=[0],
|
||||
device_id=0,
|
||||
num_blocks=1,
|
||||
block_lens=[remote_len],
|
||||
block_strides=[remote_len],
|
||||
kv_cache_layout=worker.kv_cache_layout,
|
||||
block_size=worker.block_size,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name=worker.backend_name,
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
|
||||
if rejected:
|
||||
with pytest.raises(RuntimeError, match="NVFP4"):
|
||||
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
|
||||
else:
|
||||
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
|
||||
|
||||
|
||||
# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which
|
||||
# we put here is important. First run ray, it will clean up the resources, then
|
||||
|
||||
@@ -653,6 +653,41 @@ def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
|
||||
assert exc_info.value.errno == errno.EIO
|
||||
|
||||
|
||||
def test_segmented_region_maps_files_back_to_back(iid, monkeypatch):
|
||||
"""Rows split over three shm files mapped back to back: each worker sees
|
||||
the other's writes in every file, and the creator removes every file."""
|
||||
monkeypatch.setattr(region_module, "_MIN_SEGMENT_BYTES", PAGE_SIZE)
|
||||
regions = [
|
||||
SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=5,
|
||||
rank=rank,
|
||||
kv_bytes_per_chunk=2 * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
num_segments=3,
|
||||
)
|
||||
for rank in range(2)
|
||||
]
|
||||
segments = regions[0]._segments
|
||||
try:
|
||||
assert [length // PAGE_SIZE for _, _, length in segments] == [4, 4, 2]
|
||||
for rank, region in enumerate(regions):
|
||||
ranges = list(region.iter_worker_slot_ranges())
|
||||
first_row = rank * 5 // 2
|
||||
assert ranges[0] == ((first_row * 2 + rank) * PAGE_SIZE, PAGE_SIZE)
|
||||
assert sorted(ranges) == [
|
||||
((row * 2 + rank) * PAGE_SIZE, PAGE_SIZE) for row in range(5)
|
||||
]
|
||||
region.create_next_worker_view(PAGE_SIZE).fill_(rank + 1)
|
||||
rows = regions[0].base_tensor.view(5, 2, PAGE_SIZE)
|
||||
assert (rows[:, 0] == 1).all() and (rows[:, 1] == 2).all()
|
||||
del rows
|
||||
finally:
|
||||
for region in regions:
|
||||
region.cleanup()
|
||||
assert not any(os.path.exists(path) for path, _, _ in segments)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-worker race — concurrent construction
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -899,6 +934,7 @@ def _unmapped_registration_region(num_chunks, slot_size, num_workers):
|
||||
region.cpu_page_size = slot_size
|
||||
region._row_stride = num_workers * slot_size
|
||||
region.total_size_bytes = region.num_chunks * region._row_stride
|
||||
region._segments = [("unused", 0, region.total_size_bytes)]
|
||||
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
|
||||
region.pinned_addresses = []
|
||||
region.is_pinned = False
|
||||
|
||||
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
|
||||
|
||||
|
||||
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
@@ -70,6 +72,46 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_dspark_revisited_position_uses_step_seed(use_fp64):
|
||||
"""A cached draft at a revisited position must use its current step."""
|
||||
device = torch.device("cuda")
|
||||
batch = 4096
|
||||
spec = DSparkSpeculator.__new__(DSparkSpeculator)
|
||||
spec.acceptance_estimator = None
|
||||
spec._d2t_scatter_index = None
|
||||
spec.draft_watermarker = None
|
||||
spec.use_fp64_gumbel = use_fp64
|
||||
spec.temperature = torch.ones(batch, device=device)
|
||||
spec.seeds = torch.arange(batch, dtype=torch.int64, device=device)
|
||||
spec._step_cols = torch.arange(2, dtype=torch.int32, device=device)
|
||||
spec.draft_logits = torch.empty(batch, 2, 2, device=device)
|
||||
logits = torch.tensor([0.7, 0.3], device=device).log().expand(batch, -1)
|
||||
mapping = torch.arange(batch, device=device)
|
||||
positions = torch.full((batch,), 22, dtype=torch.int64, device=device)
|
||||
|
||||
first = spec._sample_logits(logits, mapping, positions, 1)
|
||||
repeated = spec._sample_logits(logits, mapping, positions, 1)
|
||||
next_round = spec._sample_logits(logits, mapping, positions, 0)
|
||||
torch.testing.assert_close(first, repeated)
|
||||
expected = gumbel_sample(
|
||||
logits,
|
||||
mapping,
|
||||
spec.temperature,
|
||||
spec.seeds,
|
||||
positions - 1,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=spec.draft_logits,
|
||||
logits_cache_col=spec._step_cols[0],
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
torch.testing.assert_close(next_round, expected)
|
||||
mismatch_rate = (first != next_round).float().mean().item()
|
||||
assert 0.35 < mismatch_rate < 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
|
||||
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
|
||||
monkeypatch, cuda_alike_platform
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers import logits_processor
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.models.lilicorr import LiLiCorrHead
|
||||
from vllm.transformers_utils.configs.eagle import EAGLEConfig
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler
|
||||
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator
|
||||
|
||||
@@ -500,6 +501,60 @@ def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic):
|
||||
assert cache[1].isneginf().all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_candidate_walk_revisited_position_matches_step_seed(use_fp64):
|
||||
"""The candidate walk must use the same step key as ordinary draft sampling."""
|
||||
device = torch.device("cuda")
|
||||
batch, steps, top_k = 4096, 2, 2
|
||||
sampler = CandidateSampler(batch, steps, top_k, device)
|
||||
candidates = torch.arange(top_k, device=device).expand(batch, steps, top_k)
|
||||
logits = torch.tensor([0.7, 0.3], device=device).log()
|
||||
scores = logits.expand(batch, steps, top_k, top_k)
|
||||
mapping = torch.arange(batch, device=device)
|
||||
temperature = torch.ones(batch, device=device)
|
||||
seeds = mapping.to(torch.int64)
|
||||
positions = torch.arange(21, 21 + steps, device=device).repeat(batch)
|
||||
tokens = torch.empty(batch, steps, dtype=torch.int64, device=device)
|
||||
cache = torch.empty(batch, steps, top_k, device=device)
|
||||
first_round = None
|
||||
for start in (21, 22):
|
||||
positions.view(batch, steps).copy_(
|
||||
torch.arange(start, start + steps, device=device)
|
||||
)
|
||||
sampler.sample(
|
||||
candidates,
|
||||
scores,
|
||||
batch,
|
||||
positions,
|
||||
mapping.repeat_interleave(steps),
|
||||
temperature,
|
||||
seeds,
|
||||
tokens,
|
||||
cache,
|
||||
use_fp64,
|
||||
)
|
||||
for step in range(steps):
|
||||
expected = gumbel_sample(
|
||||
logits.expand(batch, -1),
|
||||
mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
positions.view(batch, steps)[:, step] - 1,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=torch.tensor(step, device=device),
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
torch.testing.assert_close(tokens[:, step], expected)
|
||||
if first_round is None:
|
||||
first_round = tokens.clone()
|
||||
else:
|
||||
mismatch_rate = (first_round[:, 1] != tokens[:, 0]).float().mean().item()
|
||||
assert 0.35 < mismatch_rate < 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("convolution", "mismatch", "quantized"),
|
||||
[
|
||||
|
||||
@@ -114,6 +114,10 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
|
||||
def reasoning_end_token_ids(self):
|
||||
return self._end_token_ids
|
||||
|
||||
@property
|
||||
def implicit_reasoning_end_token_ids(self):
|
||||
return frozenset()
|
||||
|
||||
def find_reasoning_end(self, token_ids):
|
||||
self.windows.append(list(token_ids))
|
||||
for offset, token in enumerate(token_ids):
|
||||
|
||||
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
@pytest.mark.parametrize("probabilistic_draft", [False, True])
|
||||
def test_multiround_rejection_sample_is_unbiased(
|
||||
use_block_verification: bool, probabilistic_draft: bool
|
||||
):
|
||||
"""Uncommitted positions must receive fresh noise on later verification rounds."""
|
||||
torch.manual_seed(42)
|
||||
num_trials, num_tokens, k = 32768, 6, 2
|
||||
device = "cuda"
|
||||
target_probs = torch.tensor(
|
||||
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
|
||||
)
|
||||
draft_logits = torch.tensor(
|
||||
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
|
||||
).log()
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_probs.log(), draft_logits, k, 1.0, num_trials
|
||||
)
|
||||
seeds = inputs["seed"]
|
||||
output = torch.full((num_trials, num_tokens + k), -1, device=device)
|
||||
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
|
||||
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
|
||||
accepted = 0
|
||||
while (lengths < num_tokens).any():
|
||||
active = (lengths < num_tokens).nonzero().flatten()
|
||||
n = active.numel()
|
||||
starts = lengths[active]
|
||||
local_positions = torch.arange(k + 1, device=device).repeat(n)
|
||||
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
|
||||
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
|
||||
draft_sampled[:, 0] = output[active, starts - 1]
|
||||
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
|
||||
if probabilistic_draft:
|
||||
for step in range(k):
|
||||
draft_sampled[:, step + 1] = gumbel_sample(
|
||||
draft_logits.expand(n, -1),
|
||||
active,
|
||||
inputs["temperature"],
|
||||
seeds,
|
||||
starts + step,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=torch.tensor(step, device=device),
|
||||
)
|
||||
sampled, counts = rejection_sample(
|
||||
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
|
||||
draft_logits=logits_cache,
|
||||
draft_sampled=draft_sampled.flatten(),
|
||||
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
|
||||
* (k + 1),
|
||||
pos=positions,
|
||||
idx_mapping=active,
|
||||
expanded_idx_mapping=active.repeat_interleave(k + 1),
|
||||
expanded_local_pos=local_positions,
|
||||
temperature=inputs["temperature"],
|
||||
seed=seeds,
|
||||
num_speculative_steps=k,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
for column in range(k + 1):
|
||||
valid = counts > column
|
||||
output[active[valid], starts[valid] + column] = sampled[valid, column]
|
||||
lengths[active] += counts
|
||||
accepted += (counts - 1).sum().item()
|
||||
|
||||
assert accepted > num_trials, "The test must exercise accepted draft tokens."
|
||||
for position in range(num_tokens):
|
||||
_assert_distribution_match(
|
||||
output[:, position], target_probs, device, label=f"position {position}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
|
||||
def test_greedy_rejection_sample(num_speculative_steps: int):
|
||||
"""Verify that greedy (temperature=0) always outputs the target argmax
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest import mock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -198,3 +199,52 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
|
||||
assert queue.get_nowait() is STREAM_FINISHED
|
||||
llm._add_request.assert_not_awaited()
|
||||
|
||||
|
||||
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
llm.model_config = MagicMock()
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.has_request.return_value = has_request
|
||||
|
||||
# The first processed request is the final (end-of-stream) request.
|
||||
reqs = [MagicMock(prompt_embeds=None) for _ in range(num_chunks + 1)]
|
||||
reqs[0].request_id = "r1-int"
|
||||
llm.input_processor = MagicMock()
|
||||
llm.input_processor.process_inputs.side_effect = reqs
|
||||
return llm, reqs
|
||||
|
||||
|
||||
async def _stream_chunks(llm: AsyncLLM, num_chunks: int) -> None:
|
||||
async def chunks():
|
||||
for i in range(num_chunks):
|
||||
yield StreamingInput(prompt=f"frame-{i}")
|
||||
|
||||
with mock.patch(
|
||||
"vllm.v1.engine.async_llm.extract_prompt_components",
|
||||
return_value=("", None, None),
|
||||
):
|
||||
queue = await llm._add_streaming_input_request(
|
||||
"r1", chunks(), SamplingParams(max_tokens=4)
|
||||
)
|
||||
task = queue._input_stream_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("session_live", [True, False])
|
||||
async def test_handle_inputs_stops_after_session_finished(session_live: bool):
|
||||
"""Once the session's request state is gone (aborted or errored), no
|
||||
further chunks or final request are sent: they would recreate a request
|
||||
state that is never cleaned up."""
|
||||
llm, reqs = _make_streaming_llm(num_chunks=2, has_request=session_live)
|
||||
await _stream_chunks(llm, num_chunks=2)
|
||||
|
||||
sent = [call.args[0] for call in llm._add_request.await_args_list]
|
||||
# Both chunks then the final request, or only the first chunk.
|
||||
assert sent == ([reqs[1], reqs[2], reqs[0]] if session_live else [reqs[1]])
|
||||
|
||||
@@ -5,9 +5,11 @@ import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
|
||||
from vllm.config import DeviceConfig, VllmConfig
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -28,7 +30,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.outputs import LogprobsLists, ModelRunnerOutput
|
||||
from vllm.v1.outputs import DraftTokenIds, LogprobsLists, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
@@ -676,3 +678,41 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
cached_state_cycle1["prompt_token_ids"]
|
||||
is not cached_state_cycle3["prompt_token_ids"]
|
||||
), "Cached states from different cycles should be independent objects."
|
||||
|
||||
|
||||
def _model_output(request_id: str, token_ids: list[int]) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[request_id],
|
||||
req_id_to_index={request_id: 0},
|
||||
sampled_token_ids=[token_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={request_id: None},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling):
|
||||
"""A stop on an accepted draft trims the step's later tokens. The next
|
||||
chunk must still be computed in full rather than landing on positions
|
||||
whose KV belongs to the trimmed drafts."""
|
||||
scheduler = create_spec_scheduler(
|
||||
num_speculative_tokens=2,
|
||||
async_scheduling=async_scheduling,
|
||||
speculative_method="ngram_gpu" if async_scheduling else None,
|
||||
)
|
||||
session = DummyRequest("session", prompt_token_ids=[1, 2, 3], max_tokens=2)
|
||||
scheduler.add_request(session)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
|
||||
scheduler.update_draft_token_ids(DraftTokenIds(["session"], [[11, 12]]))
|
||||
# Both drafts are accepted, but max_tokens stops at 11.
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(), _model_output("session", [11, 12, 13])
|
||||
)
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
scheduler.add_request(DummyRequest("session", prompt_token_ids=[7, 8]))
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
|
||||
assert output.num_scheduled_tokens["session"] == 2
|
||||
|
||||
@@ -6,6 +6,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from threading import Event
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -72,20 +73,26 @@ class MockReasoner:
|
||||
|
||||
|
||||
class MockEngineReasoner(ParserEngineReasoningAdapter):
|
||||
"""Ends reasoning implicitly on ``marker``, which is also content."""
|
||||
"""Ends reasoning implicitly on ``marker``, which is also content, and
|
||||
explicitly on ``end_marker`` when one is given."""
|
||||
|
||||
def __init__(self, tokenizer, marker: int):
|
||||
def __init__(self, tokenizer, marker: int, end_marker: int | None = None):
|
||||
self.marker = marker
|
||||
self.end_marker = end_marker
|
||||
|
||||
@property
|
||||
def reasoning_end_token_ids(self):
|
||||
return frozenset({self.marker, self.end_marker} - {None})
|
||||
|
||||
@property
|
||||
def implicit_reasoning_end_token_ids(self):
|
||||
return frozenset({self.marker})
|
||||
|
||||
def find_reasoning_end(self, token_ids):
|
||||
ids = list(token_ids)
|
||||
if self.marker in ids:
|
||||
return ReasoningEnd(ids.index(self.marker), True)
|
||||
return ReasoningEnd(len(ids), False)
|
||||
for offset, token in enumerate(token_ids):
|
||||
if token in self.reasoning_end_token_ids:
|
||||
return ReasoningEnd(offset, token == self.marker)
|
||||
return ReasoningEnd(len(token_ids), False)
|
||||
|
||||
|
||||
def _single_token(tokenizer, text: str) -> int:
|
||||
@@ -122,6 +129,7 @@ def _build_harness(
|
||||
enable_in_reasoning: bool = False,
|
||||
reasoning_parser_kwargs: dict | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
num_spec_tokens: int = NUM_SPEC_TOKENS,
|
||||
) -> tuple[StructuredOutputManager, Request]:
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(tokenizer=TOKENIZER),
|
||||
@@ -132,8 +140,10 @@ def _build_harness(
|
||||
),
|
||||
speculative_config=SpeculativeConfig(
|
||||
model="[ngram]",
|
||||
num_speculative_tokens=NUM_SPEC_TOKENS,
|
||||
),
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
)
|
||||
if num_spec_tokens
|
||||
else None,
|
||||
)
|
||||
manager = StructuredOutputManager(vllm_config)
|
||||
if use_reasoner:
|
||||
@@ -171,8 +181,20 @@ def _build_harness(
|
||||
return manager, request
|
||||
|
||||
|
||||
def _row_pattern(bitmask) -> str:
|
||||
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
|
||||
def _row_pattern(bitmask, masked_ids=()) -> str:
|
||||
"""U: unconstrained; R: all but `masked_ids` allowed; C: grammar rows."""
|
||||
reasoning_row = np.full(bitmask.shape[1], -1, dtype=np.int32)
|
||||
for token_id in masked_ids:
|
||||
reasoning_row.view(np.uint32)[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
|
||||
|
||||
def kind(row) -> str:
|
||||
if (row == -1).all():
|
||||
return "U"
|
||||
if masked_ids and (row == reasoning_row).all():
|
||||
return "R"
|
||||
return "C"
|
||||
|
||||
return "".join(kind(row) for row in bitmask)
|
||||
|
||||
|
||||
def _run_real_flow(
|
||||
@@ -183,6 +205,7 @@ def _run_real_flow(
|
||||
expected_row_pattern: str,
|
||||
expected_reasoning: bool | None,
|
||||
expect_terminated: bool,
|
||||
masked_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
assert len(expected_validated) <= len(raw_drafts)
|
||||
|
||||
@@ -197,7 +220,7 @@ def _run_real_flow(
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert bitmask.shape[0] == len(raw_drafts) + 1
|
||||
assert _row_pattern(bitmask) == expected_row_pattern
|
||||
assert _row_pattern(bitmask, masked_ids or ()) == expected_row_pattern
|
||||
|
||||
# grammar_bitmask() must rollback any speculative state it advanced.
|
||||
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
|
||||
@@ -375,7 +398,8 @@ FLOW_CASES = [
|
||||
FlowCase(
|
||||
raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"),
|
||||
expected_validated=(" ",),
|
||||
expected_row_pattern="UUUUU",
|
||||
# The grammar cannot start with the marker, so reasoning rows mask it.
|
||||
expected_row_pattern="RRUUU",
|
||||
expected_reasoning=False,
|
||||
expect_terminated=False,
|
||||
engine_reasoner=True,
|
||||
@@ -421,9 +445,93 @@ def test_real_flow(
|
||||
expected_row_pattern=case.expected_row_pattern,
|
||||
expected_reasoning=case.expected_reasoning,
|
||||
expect_terminated=case.expect_terminated,
|
||||
masked_ids=[reasoner_kwargs["marker"]] if case.engine_reasoner else None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("structured_outputs", "raw_drafts", "expected_row_pattern"),
|
||||
[
|
||||
# JSON cannot start with the marker: reasoning rows mask it and keep
|
||||
# every other token, including the explicit end, allowed.
|
||||
pytest.param(
|
||||
StructuredOutputsParams(json=JSON_SCHEMA), (" ", " "), "RRR", id="json"
|
||||
),
|
||||
# The grammar can start with the marker: it stays allowed and, once
|
||||
# sampled, is the first token the grammar consumes.
|
||||
pytest.param(
|
||||
StructuredOutputsParams(regex="z[0-9]+"),
|
||||
(" ", IMPLICIT_THINK_END, "1"),
|
||||
"UUCC",
|
||||
id="regex_starts_with_marker",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_reasoning_rows_mask_grammar_invalid_implicit_end(
|
||||
tokenizer,
|
||||
backend: StructuredOutputsBackend,
|
||||
structured_outputs: StructuredOutputsParams,
|
||||
raw_drafts: tuple[str, ...],
|
||||
expected_row_pattern: str,
|
||||
):
|
||||
marker = _single_token(tokenizer, IMPLICIT_THINK_END)
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
reasoning_ended=False,
|
||||
reasoning_parser_kwargs={
|
||||
"marker": marker,
|
||||
"end_marker": _single_token(tokenizer, THINK_END),
|
||||
},
|
||||
structured_outputs=structured_outputs,
|
||||
)
|
||||
manager.reasoner_cls = MockEngineReasoner
|
||||
drafts = _to_token_ids(tokenizer, raw_drafts)
|
||||
assert manager.validate_tokens(request, drafts) == drafts
|
||||
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={request.request_id: drafts},
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert _row_pattern(bitmask, [marker]) == expected_row_pattern
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
def test_parallel_fill_masks_grammar_invalid_implicit_end(
|
||||
tokenizer, backend: StructuredOutputsBackend
|
||||
):
|
||||
marker, end = _to_token_ids(tokenizer, (IMPLICIT_THINK_END, THINK_END))
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
reasoning_ended=False,
|
||||
reasoning_parser_kwargs={"marker": marker, "end_marker": end},
|
||||
num_spec_tokens=0,
|
||||
)
|
||||
manager.reasoner_cls = MockEngineReasoner
|
||||
|
||||
def row_pattern() -> str:
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
assert bitmask is not None
|
||||
return _row_pattern(bitmask, [marker])
|
||||
|
||||
manager.fill_bitmask_parallel_threshold = 0
|
||||
manager.fill_bitmask_parallel_batch_size = 16
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
manager.executor_for_fillmask = executor
|
||||
assert row_pattern() == "R"
|
||||
request.append_output_token_ids([end])
|
||||
assert manager.accept_tokens(request, [end])
|
||||
assert row_pattern() == "C"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
|
||||
|
||||
@@ -9,12 +9,14 @@ never addressed by the logical view.
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
@@ -403,49 +405,63 @@ class _FakeSharedHostRegion:
|
||||
self.cleanup_calls += 1
|
||||
|
||||
|
||||
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
|
||||
"""TP-shared profiling pools must use region-aware chunk cleanup."""
|
||||
region = _FakeSharedHostRegion()
|
||||
def test_profiling_cleanup_keeps_host_pool_for_real_kv_cache(monkeypatch):
|
||||
"""The real KV cache must reuse the profiling host pool, not pin it again."""
|
||||
pinned = torch.empty(8, dtype=torch.int8)
|
||||
allocate_pinned = MagicMock(return_value=(pinned, pinned))
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "allocate_pinned_host_pool", allocate_pinned
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "check_hisparse_host_memory", lambda size: None
|
||||
)
|
||||
vllm_config = SimpleNamespace(instance_id="profiling-reuse")
|
||||
kv_cache_config = SimpleNamespace(hisparse_shared_host_pool=False)
|
||||
pool = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
backing = pool.allocate(8)
|
||||
runtime = SimpleNamespace(
|
||||
_host_cache=object(),
|
||||
registered_host_pool=region.base_tensor,
|
||||
hot_backing=object(),
|
||||
shared_host_region=region,
|
||||
_host_cache=backing, registered_host_pool=pinned, hot_backing=object()
|
||||
)
|
||||
forward_context = {
|
||||
"layer": SimpleNamespace(
|
||||
hisparse_cache=SimpleNamespace(runtime=runtime),
|
||||
)
|
||||
"layer": SimpleNamespace(hisparse_cache=SimpleNamespace(runtime=runtime))
|
||||
}
|
||||
released = []
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
released.append((runtimes, pinned_host_pools, shared_host_region))
|
||||
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module,
|
||||
"release_pinned_state",
|
||||
release_pinned_state,
|
||||
)
|
||||
|
||||
attn_utils_module.release_hisparse_profiling_cache(forward_context)
|
||||
|
||||
assert released == [([runtime], [], region)]
|
||||
assert not hasattr(runtime, "_host_cache")
|
||||
reused = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
assert reused is pool
|
||||
assert reused.allocate(8) is backing
|
||||
allocate_pinned.assert_called_once_with(8)
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
|
||||
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
"""A failure after mmap allocation must not leak the shared registration."""
|
||||
@pytest.mark.parametrize("shared", [False, True])
|
||||
def test_init_hisparse_rolls_back_host_pool(monkeypatch, failure_phase, shared):
|
||||
"""Failed initialization must release shared or private registrations."""
|
||||
region = _FakeSharedHostRegion()
|
||||
registered = torch.empty(4096, dtype=torch.int8)
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostUnregister.return_value.value = 0
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1),
|
||||
instance_id="rollback",
|
||||
)
|
||||
pools = []
|
||||
|
||||
def allocate(*args):
|
||||
args[-1].shared_region = region
|
||||
pools.append(args[-1])
|
||||
if shared:
|
||||
args[-1].shared_region = region
|
||||
else:
|
||||
args[-1].registered = registered
|
||||
args[-1].backing = registered[:8]
|
||||
if failure_phase == "allocation":
|
||||
raise RuntimeError("initialization failed")
|
||||
return {}
|
||||
@@ -472,7 +488,13 @@ def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
{},
|
||||
SimpleNamespace(),
|
||||
)
|
||||
assert region.cleanup_calls == 1
|
||||
assert region.cleanup_calls == int(shared)
|
||||
if shared:
|
||||
cudart.cudaHostUnregister.assert_not_called()
|
||||
else:
|
||||
cudart.cudaHostUnregister.assert_called_once_with(registered.data_ptr())
|
||||
assert attn_utils_module.get_hisparse_host_pool(vllm_config, None) is not pools[0]
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The CPU worker's automatic KV cache sizing must report the right cause.
|
||||
|
||||
A worker whose own footprint exceeds its reservation and a node that is short
|
||||
of free memory need opposite remedies, so they get separate errors.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.cpu_resource_utils import MemoryNodeInfo
|
||||
|
||||
# Importing the CPU worker patches torch and the GPU buffer utilities for the
|
||||
# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into
|
||||
# the tests and they run only on the CPU platform.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform"
|
||||
)
|
||||
|
||||
GiB = 1 << 30
|
||||
|
||||
|
||||
def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB):
|
||||
import vllm.v1.worker.cpu_worker as cpu_worker_mod
|
||||
from vllm.v1.worker.cpu_worker import CPUWorker
|
||||
|
||||
worker = CPUWorker.__new__(CPUWorker)
|
||||
worker.requested_cpu_memory = reserved
|
||||
worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None)
|
||||
monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_allowed_cpu_list",
|
||||
lambda: [SimpleNamespace(numa_node=0)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_memory_node_info",
|
||||
lambda node: MemoryNodeInfo(total_memory=total, available_memory=available),
|
||||
)
|
||||
process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss))
|
||||
monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process)
|
||||
return worker
|
||||
|
||||
|
||||
def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch):
|
||||
# 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB).
|
||||
worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB))
|
||||
with pytest.raises(ValueError) as exc:
|
||||
worker.determine_available_memory()
|
||||
msg = str(exc.value)
|
||||
assert "increase `--gpu-memory-utilization` to more than 0.10" in msg
|
||||
assert "--kv-cache-memory-bytes" in msg
|
||||
# Neither the misleading remedy nor a negative KV size.
|
||||
assert "other processes" not in msg
|
||||
assert "-0." not in msg
|
||||
|
||||
|
||||
def test_short_node_memory_still_asks_to_free_memory(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB)
|
||||
with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"):
|
||||
worker.determine_available_memory()
|
||||
|
||||
|
||||
def test_budget_that_fits_sizes_the_kv_cache(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB)
|
||||
assert worker.determine_available_memory() == 2 * GiB
|
||||
@@ -0,0 +1,217 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
quantized_lm_head_copy,
|
||||
refresh_quantized_lm_heads,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
# Not a multiple of the vocab padding, so the head has padded rows.
|
||||
VOCAB, HIDDEN = 4000, 512
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="Marlin needs CUDA"
|
||||
)
|
||||
# NVFP4 needs a CUDA W4A16 kernel; ROCm only runs the fp8 cases.
|
||||
requires_nvfp4 = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
|
||||
)
|
||||
NVFP4 = pytest.param("nvfp4", marks=requires_nvfp4)
|
||||
|
||||
|
||||
def _target_head(monkeypatch, seed: int = 0) -> ParallelLMHead:
|
||||
# Online methods read the model dtype from the current config.
|
||||
model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.online.fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=model_config),
|
||||
)
|
||||
head = ParallelLMHead(VOCAB, HIDDEN, disable_tp=True)
|
||||
gen = torch.Generator().manual_seed(seed)
|
||||
head.weight.data.copy_(torch.randn(head.weight.shape, generator=gen) * 0.02)
|
||||
return head.to("cuda", torch.bfloat16)
|
||||
|
||||
|
||||
def _tensors(module: torch.nn.Module) -> dict[str, torch.Tensor]:
|
||||
return dict(module.named_parameters()) | dict(module.named_buffers())
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
("quantization", "max_rel_err", "min_top1"),
|
||||
[("fp8", 0.05, 0.9), pytest.param("nvfp4", 0.15, 0.75, marks=requires_nvfp4)],
|
||||
)
|
||||
def test_tracks_bf16_head_through_logits_processor(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization, max_rel_err, min_top1
|
||||
):
|
||||
head = _target_head(monkeypatch)
|
||||
target_weight = head.weight.detach().clone()
|
||||
draft_head = quantized_lm_head_copy(head, quantization)
|
||||
|
||||
assert isinstance(draft_head, ParallelLMHead)
|
||||
# The source head keeps its bf16 weight, and no bf16 copy is kept around.
|
||||
assert torch.equal(head.weight, target_weight)
|
||||
assert draft_head.weight.dtype != torch.bfloat16
|
||||
|
||||
logits_processor = LogitsProcessor(VOCAB)
|
||||
hidden = torch.randn(2048, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
ref = logits_processor(head, hidden).float()
|
||||
out = logits_processor(draft_head, hidden).float()
|
||||
assert out.shape == ref.shape == (2048, VOCAB)
|
||||
rel_err = ((out - ref).norm() / ref.norm()).item()
|
||||
top1 = (out.argmax(-1) == ref.argmax(-1)).float().mean().item()
|
||||
assert rel_err < max_rel_err
|
||||
assert top1 > min_top1
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
|
||||
def test_refresh_follows_target_update(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization
|
||||
):
|
||||
"""After the target's weights change, the copy is re-derived in place: it
|
||||
equals a fresh quantization of the new target, keeps its storage, and
|
||||
frees the temporary copy without waiting for the cycle collector."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
drafter.lm_head = quantized_lm_head_copy(head, quantization)
|
||||
ptrs = {k: t.data_ptr() for k, t in _tensors(drafter.lm_head).items()}
|
||||
|
||||
new_weight = _target_head(monkeypatch, seed=1).weight.detach()
|
||||
head.weight.data.copy_(new_weight)
|
||||
gc.disable()
|
||||
try:
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
refresh_quantized_lm_heads(drafter)
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
expected = _tensors(quantized_lm_head_copy(head, quantization))
|
||||
for name, tensor in _tensors(drafter.lm_head).items():
|
||||
assert tensor.data_ptr() == ptrs[name]
|
||||
if name != "workspace":
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
assert torch.equal(head.weight, new_weight)
|
||||
|
||||
|
||||
def _draft_update_worker(vllm_config, drafter: torch.nn.Module, **engine) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.vllm_config = vllm_config
|
||||
worker.weight_transfer_engine = SimpleNamespace(
|
||||
supports_draft_weight_update=True,
|
||||
reset_weight_update_target=lambda: None,
|
||||
**engine,
|
||||
)
|
||||
worker.model_runner = SimpleNamespace(get_draft_model=lambda: drafter)
|
||||
worker._weight_update_active = False
|
||||
worker._set_draft_weight_update_target = lambda: None
|
||||
return worker
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_draft_weight_update_refreshes_copy(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""A draft weight update can write a target weight the drafter shares, such
|
||||
as an embedding tied to the lm_head; the copy must follow it."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
drafter.lm_head = quantized_lm_head_copy(head, "fp8")
|
||||
worker = _draft_update_worker(
|
||||
default_vllm_config,
|
||||
drafter,
|
||||
start_weight_update=lambda: None,
|
||||
finish_weight_update=lambda: None,
|
||||
)
|
||||
|
||||
worker.start_draft_weight_update()
|
||||
head.weight.data.copy_(_target_head(monkeypatch, seed=1).weight)
|
||||
worker.finish_weight_update()
|
||||
|
||||
expected = _tensors(quantized_lm_head_copy(head, "fp8"))
|
||||
for name, tensor in _tensors(drafter.lm_head).items():
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
|
||||
def test_draft_weight_update_loads_lm_head(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization
|
||||
):
|
||||
"""A draft update carrying lm_head.weight loads the shared target head, as
|
||||
without quantization, and the copy is re-derived from it in place."""
|
||||
head = _target_head(monkeypatch)
|
||||
record_metadata_for_reloading(head)
|
||||
drafter = torch.nn.Module()
|
||||
draft_head = quantized_lm_head_copy(head, quantization)
|
||||
drafter.lm_head = draft_head
|
||||
ptrs = {k: t.data_ptr() for k, t in _tensors(draft_head).items()}
|
||||
new_weight = torch.randn(VOCAB, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def load(update_info):
|
||||
param = drafter.get_parameter("lm_head.weight")
|
||||
getattr(param, "weight_loader", default_weight_loader)(param, new_weight)
|
||||
|
||||
worker = _draft_update_worker(
|
||||
default_vllm_config,
|
||||
drafter,
|
||||
start_weight_update=lambda: initialize_layerwise_reload(drafter),
|
||||
update_weights=load,
|
||||
finish_weight_update=lambda: finalize_layerwise_reload(drafter, None),
|
||||
)
|
||||
worker.start_draft_weight_update()
|
||||
worker.update_weights({})
|
||||
worker.finish_weight_update()
|
||||
|
||||
assert drafter.lm_head is draft_head
|
||||
assert torch.equal(head.weight[:VOCAB], new_weight)
|
||||
expected = _tensors(quantized_lm_head_copy(head, quantization))
|
||||
for name, tensor in _tensors(draft_head).items():
|
||||
assert tensor.data_ptr() == ptrs[name]
|
||||
if name != "workspace":
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
hidden = torch.randn(64, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
assert LogitsProcessor(VOCAB)(draft_head, hidden).isfinite().all()
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@requires_nvfp4
|
||||
def test_layerwise_reload_of_drafter_keeps_copy(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""A drafter weight update reloads the drafter layerwise; the derived copy
|
||||
must come back unchanged, not be re-quantized."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
record_metadata_for_reloading(drafter)
|
||||
drafter.lm_head = quantized_lm_head_copy(head, "nvfp4")
|
||||
before = {k: t.clone() for k, t in _tensors(drafter.lm_head).items()}
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("the copy was re-quantized")
|
||||
|
||||
monkeypatch.setattr(
|
||||
drafter.lm_head.quant_method, "process_weights_after_loading", fail
|
||||
)
|
||||
initialize_layerwise_reload(drafter)
|
||||
finalize_layerwise_reload(drafter, None)
|
||||
|
||||
after = _tensors(drafter.lm_head)
|
||||
assert after.keys() == before.keys()
|
||||
for name, tensor in before.items():
|
||||
assert torch.equal(after[name], tensor), name
|
||||
@@ -302,10 +302,8 @@ def _sync_dp(
|
||||
reduced[4] = int(allow_ubatching)
|
||||
reduced[5] = 8 # num_reqs
|
||||
|
||||
with (
|
||||
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
|
||||
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
|
||||
):
|
||||
dp_group = SimpleNamespace(cpu_all_reduce_=lambda t: t.copy_(reduced))
|
||||
with patch.object(dp_utils, "get_dp_group", lambda: dp_group):
|
||||
return dp_utils.sync_cudagraph_and_dp_padding(
|
||||
cudagraph_manager=cudagraph_manager,
|
||||
desired_batch_desc=BatchExecutionDescriptor(
|
||||
|
||||
@@ -65,6 +65,9 @@ class _RecordingModelRunner:
|
||||
def reset_lora_state(self) -> None:
|
||||
self.reset_lora_calls += 1
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
return None
|
||||
|
||||
|
||||
def _make_worker(engine: _RecordingEngine | None) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
|
||||
@@ -62,7 +62,7 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_binding,
|
||||
"HiSparseHostPool",
|
||||
"get_hisparse_host_pool",
|
||||
lambda vllm_config, kv_cache_config: host_pool,
|
||||
)
|
||||
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
|
||||
|
||||
@@ -1257,6 +1257,8 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.cache_handles = []
|
||||
worker.pinned_host_pools = []
|
||||
worker.shared_host_region = object()
|
||||
worker.vllm_config = SimpleNamespace(instance_id="shutdown")
|
||||
pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
released = False
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
@@ -1273,6 +1275,9 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.shutdown()
|
||||
|
||||
assert released
|
||||
new_pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
assert new_pool is not pool
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(worker.vllm_config)
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
|
||||
@@ -253,6 +253,8 @@ def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
|
||||
positions,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
)
|
||||
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
|
||||
expected = torch.where(skip_mask, ordinary, watermarked)
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.v1.worker.gpu.sample.watermark import (
|
||||
philox_gumbel_sample,
|
||||
repeated_context_mask,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
@@ -254,8 +255,8 @@ def test_speculative_decoding_uses_fixed_dual_key_roles():
|
||||
assert draft is not None
|
||||
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
|
||||
assert draft.num_speculative_steps == 3
|
||||
assert draft.deduplicate_contexts == "all"
|
||||
assert draft.deduplicate_contexts_max_history is None
|
||||
assert draft.state.deduplicate_contexts == "all"
|
||||
assert draft.state.deduplicate_contexts_max_history is None
|
||||
|
||||
|
||||
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
|
||||
@@ -1324,8 +1325,8 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
deduplicate_contexts="none",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
|
||||
draft_watermarker.enabled = torch.tensor([True, False])
|
||||
draft_watermarker.state.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
|
||||
draft_watermarker.state.enabled = torch.tensor([True, False])
|
||||
speculator.draft_watermarker = draft_watermarker
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
@@ -1343,7 +1344,7 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 4]))
|
||||
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
|
||||
assert torch.equal(draft_watermarker.state.contexts, torch.tensor([[2, 7], [4, 4]]))
|
||||
|
||||
|
||||
def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch):
|
||||
@@ -1602,11 +1603,12 @@ def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
|
||||
[True, True],
|
||||
]
|
||||
assert torch.equal(
|
||||
draft_watermarker.prior_contexts[:num_reqs].cpu(),
|
||||
draft_watermarker.state.prior_contexts[:num_reqs].cpu(),
|
||||
torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]),
|
||||
)
|
||||
assert torch.equal(
|
||||
draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]])
|
||||
draft_watermarker.state.contexts[:num_reqs].cpu(),
|
||||
torch.tensor([[6, 6], [6, 6]]),
|
||||
)
|
||||
for step, logits in enumerate(step_logits):
|
||||
# The 0-d draft step selects the logits cache column.
|
||||
@@ -1661,6 +1663,154 @@ def test_draft_watermarker_cuda_graph_reads_current_prompt_lens():
|
||||
assert mask.item()
|
||||
|
||||
|
||||
def test_dflash_draft_sampler_watermarks_drafts_in_step_order(monkeypatch):
|
||||
class NextTokenWatermarker:
|
||||
context_width = 2
|
||||
|
||||
@staticmethod
|
||||
def _try_sample_block(*args, **kwargs):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def sample(logits, contexts, random_sampler=None, skip_mask=None):
|
||||
watermarked = contexts[:, -1] + 1
|
||||
return WatermarkSample(
|
||||
torch.where(skip_mask, random_sampler(logits), watermarked), logits
|
||||
)
|
||||
|
||||
def ordinary_sample(logits, idx_mapping, temperature, seed, pos, **kwargs):
|
||||
assert kwargs["use_fp64"]
|
||||
cache, col = kwargs["logits_cache"], kwargs["logits_cache_col"]
|
||||
cache[idx_mapping, col] = kwargs["logits_cache_source"]
|
||||
return seed[idx_mapping] + pos
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample", ordinary_sample
|
||||
)
|
||||
num_reqs, num_steps, vocab_size = 3, 2, 4
|
||||
draft_watermarker = DraftWatermarker(
|
||||
NextTokenWatermarker(),
|
||||
max_num_reqs=8,
|
||||
device=torch.device("cpu"),
|
||||
num_speculative_steps=num_steps,
|
||||
deduplicate_contexts="none",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
# Batch rows hold request slots 2, 0, 1. Slot 0 is greedy and row 2 opted out.
|
||||
draft_watermarker.state.contexts[:num_reqs] = torch.tensor([[1, 2], [3, 4], [5, 6]])
|
||||
draft_watermarker.state.enabled[:num_reqs] = torch.tensor([True, True, False])
|
||||
speculator = object.__new__(DFlashSpeculator)
|
||||
speculator.model = SimpleNamespace(
|
||||
compute_logits=lambda hidden_states: hidden_states
|
||||
)
|
||||
speculator.draft_watermarker = draft_watermarker
|
||||
speculator.use_fp64_gumbel = True
|
||||
speculator.acceptance_estimator = None
|
||||
batch_slots = torch.tensor([2, 0, 1])
|
||||
logits = torch.arange(num_reqs * num_steps * vocab_size, dtype=torch.float32)
|
||||
draft_logits = torch.zeros(num_reqs, num_steps, vocab_size)
|
||||
|
||||
sampled = speculator.sample_draft(
|
||||
hidden_states=logits.view(-1, vocab_size),
|
||||
sample_src_positions=torch.arange(num_reqs * num_steps),
|
||||
idx_mapping=batch_slots.repeat_interleave(num_steps),
|
||||
temperature=torch.tensor([0.0, 1.0, 0.5]),
|
||||
seeds=torch.tensor([100, 200, 300]),
|
||||
draft_step=torch.arange(num_steps).repeat(num_reqs),
|
||||
draft_logits=draft_logits,
|
||||
num_steps=num_steps,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 4, 102, 103, 204, 205]))
|
||||
assert torch.equal(draft_logits[batch_slots], logits.view(num_reqs, num_steps, -1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("deduplicate_contexts", ["none", "single_turn", "all"])
|
||||
def test_dflash_draft_sampler_matches_step_loop_on_cuda(deduplicate_contexts, dtype):
|
||||
"""The fused CUDA `sample_block` is bit-exact with sampling step by step.
|
||||
|
||||
Covers repeated contexts, a NaN logit, and greedy, opted-out and padding
|
||||
rows; padding rows are discarded, so only valid rows are compared.
|
||||
"""
|
||||
num_reqs, num_steps, vocab_size, max_num_reqs = 4, 5, 1500, 6
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
logits = torch.randn(num_reqs, num_steps, vocab_size, generator=generator) * 3
|
||||
logits[..., :3] += 6
|
||||
logits[0, 1, 5] = float("nan")
|
||||
logits = logits.to(dtype).cuda()
|
||||
idx_mapping = torch.tensor([4, 2, 0, -1], dtype=torch.int32)
|
||||
valid = (idx_mapping >= 0).cuda()
|
||||
idx_mapping = idx_mapping.repeat_interleave(num_steps).cuda()
|
||||
temperature = torch.tensor([0.7, 1.0, 0.0, 1.3, 0.6, 0.9]).cuda()
|
||||
seeds = torch.arange(max_num_reqs).cuda() * 1000
|
||||
pos = (torch.arange(num_steps).repeat(num_reqs) + 20).cuda()
|
||||
draft_step = torch.arange(num_steps).repeat(num_reqs).cuda()
|
||||
contexts = torch.randint(0, 3, (num_reqs, 2), generator=generator)
|
||||
contexts[0, 0] = -1
|
||||
all_token_ids = torch.randint(0, 3, (max_num_reqs, 32), generator=generator)
|
||||
|
||||
def run(fused: bool):
|
||||
draft_watermarker = DraftWatermarker(
|
||||
GumbelWatermarker(key=42, context_width=2),
|
||||
max_num_reqs=max_num_reqs,
|
||||
device=torch.device("cuda"),
|
||||
num_speculative_steps=num_steps,
|
||||
deduplicate_contexts=deduplicate_contexts,
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
draft_watermarker.prepare(
|
||||
contexts=contexts.cuda(),
|
||||
enabled=torch.tensor([True, False, True, True]).cuda(),
|
||||
all_token_ids=all_token_ids.cuda(),
|
||||
prompt_lens=torch.full((max_num_reqs,), 4).cuda(),
|
||||
total_lens=torch.full((max_num_reqs,), 24).cuda(),
|
||||
)
|
||||
draft_logits = torch.zeros(max_num_reqs, num_steps, vocab_size, dtype=dtype)
|
||||
draft_logits = draft_logits.cuda()
|
||||
args = (temperature, seeds)
|
||||
if fused:
|
||||
sampled = draft_watermarker.sample_block(
|
||||
logits.view(-1, vocab_size),
|
||||
idx_mapping,
|
||||
*args,
|
||||
pos,
|
||||
logits_cache=draft_logits,
|
||||
logits_cache_col=draft_step,
|
||||
)
|
||||
else:
|
||||
by_step = [t.view(num_reqs, num_steps) for t in (idx_mapping, pos)]
|
||||
steps = draft_step.view(num_reqs, num_steps)
|
||||
sampled = torch.stack(
|
||||
[
|
||||
draft_watermarker.sample(
|
||||
logits[:, step],
|
||||
by_step[0][:, step],
|
||||
*args,
|
||||
by_step[1][:, step],
|
||||
logits_cache=draft_logits,
|
||||
logits_cache_col=steps[:, step],
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
)
|
||||
for step in range(num_steps)
|
||||
],
|
||||
dim=1,
|
||||
).view(-1)
|
||||
return (
|
||||
sampled.view(num_reqs, num_steps)[valid],
|
||||
draft_watermarker.state.contexts[:num_reqs][valid],
|
||||
draft_watermarker.state.prior_contexts,
|
||||
draft_logits,
|
||||
)
|
||||
|
||||
for fused, reference in zip(run(fused=True), run(fused=False)):
|
||||
torch.testing.assert_close(fused, reference, rtol=0, atol=0, equal_nan=True)
|
||||
|
||||
|
||||
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
|
||||
speculator = object.__new__(DSparkSpeculator)
|
||||
speculator.draft_logits = torch.zeros(2, 1, 8)
|
||||
|
||||
Reference in New Issue
Block a user