[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -136,8 +136,7 @@ async def _client_loop(
|
|||||||
client_id: int,
|
client_id: int,
|
||||||
):
|
):
|
||||||
"""A single async client: pulls prompts, submits to engine, consumes
|
"""A single async client: pulls prompts, submits to engine, consumes
|
||||||
hidden states as soon as each request finishes.
|
hidden states as soon as each request finishes."""
|
||||||
"""
|
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
while True:
|
while True:
|
||||||
item = await prompt_queue.get()
|
item = await prompt_queue.get()
|
||||||
|
|||||||
@@ -84,8 +84,7 @@ def make_inputs(
|
|||||||
|
|
||||||
def build_configs(head_configs, num_tokens_list, input_dtypes, tp_sizes):
|
def build_configs(head_configs, num_tokens_list, input_dtypes, tp_sizes):
|
||||||
"""Build (num_tokens, num_heads, head_size, dtype_str) config tuples,
|
"""Build (num_tokens, num_heads, head_size, dtype_str) config tuples,
|
||||||
applying TP division to num_heads and skipping invalid combos.
|
applying TP division to num_heads and skipping invalid combos."""
|
||||||
"""
|
|
||||||
configs = []
|
configs = []
|
||||||
for (_, nh, hs), nt, dtype, tp in itertools.product(
|
for (_, nh, hs), nt, dtype, tp in itertools.product(
|
||||||
head_configs, num_tokens_list, input_dtypes, tp_sizes
|
head_configs, num_tokens_list, input_dtypes, tp_sizes
|
||||||
|
|||||||
@@ -32,8 +32,7 @@ FP8_DTYPE = current_platform.fp8_dtype()
|
|||||||
|
|
||||||
def old_default_config(M, E, N, K, topk, dtype=None, block_shape=None):
|
def old_default_config(M, E, N, K, topk, dtype=None, block_shape=None):
|
||||||
"""The original defaults before https://github.com/vllm-project/vllm/pull/34846,
|
"""The original defaults before https://github.com/vllm-project/vllm/pull/34846,
|
||||||
for comparison.
|
for comparison."""
|
||||||
"""
|
|
||||||
if dtype == "fp8_w8a8" and block_shape is not None:
|
if dtype == "fp8_w8a8" and block_shape is not None:
|
||||||
return {
|
return {
|
||||||
"BLOCK_SIZE_M": 64,
|
"BLOCK_SIZE_M": 64,
|
||||||
|
|||||||
@@ -97,8 +97,7 @@ def graph_replay(fn):
|
|||||||
|
|
||||||
def calculate_diff(vocab_size, hidden_size, tp_size, num_tokens):
|
def calculate_diff(vocab_size, hidden_size, tp_size, num_tokens):
|
||||||
"""The fused kernel must be bit-exact with the path it replaces, on every
|
"""The fused kernel must be bit-exact with the path it replaces, on every
|
||||||
rank of the shard.
|
rank of the shard."""
|
||||||
"""
|
|
||||||
dtype = torch.bfloat16
|
dtype = torch.bfloat16
|
||||||
for tp_rank in range(tp_size):
|
for tp_rank in range(tp_size):
|
||||||
input_ids, weight, shard_indices = make_inputs(
|
input_ids, weight, shard_indices = make_inputs(
|
||||||
|
|||||||
@@ -75,8 +75,7 @@ class Example:
|
|||||||
|
|
||||||
If path is a file, it returns the path itself. If path is a directory, it
|
If path is a file, it returns the path itself. If path is a directory, it
|
||||||
searches for Markdown files (*.md) in the directory and returns the first one
|
searches for Markdown files (*.md) in the directory and returns the first one
|
||||||
found. If no Markdown files are found, it returns None.
|
found. If no Markdown files are found, it returns None."""
|
||||||
"""
|
|
||||||
# Single file example
|
# Single file example
|
||||||
if self.path.is_file():
|
if self.path.is_file():
|
||||||
return self.path
|
return self.path
|
||||||
@@ -91,8 +90,7 @@ class Example:
|
|||||||
"""Determine other files in the directory excluding the main file.
|
"""Determine other files in the directory excluding the main file.
|
||||||
|
|
||||||
If path is a file, it returns an empty list. Otherwise, it returns every file
|
If path is a file, it returns an empty list. Otherwise, it returns every file
|
||||||
in the directory except the main file in a list.
|
in the directory except the main file in a list."""
|
||||||
"""
|
|
||||||
# Single file example
|
# Single file example
|
||||||
if self.path.is_file():
|
if self.path.is_file():
|
||||||
return []
|
return []
|
||||||
|
|||||||
@@ -55,8 +55,7 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
class DummyPerReqLogitsProcessor:
|
class DummyPerReqLogitsProcessor:
|
||||||
"""The request-level logits processor masks out all logits except the
|
"""The request-level logits processor masks out all logits except the
|
||||||
token id identified by `target_token`
|
token id identified by `target_token`"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, target_token: int) -> None:
|
def __init__(self, target_token: int) -> None:
|
||||||
"""Specify `target_token`."""
|
"""Specify `target_token`."""
|
||||||
@@ -75,8 +74,7 @@ class DummyPerReqLogitsProcessor:
|
|||||||
|
|
||||||
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||||
"""Example of wrapping a fake request-level logit processor to create a
|
"""Example of wrapping a fake request-level logit processor to create a
|
||||||
batch-level logits processor
|
batch-level logits processor"""
|
||||||
"""
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def validate_params(cls, params: SamplingParams):
|
def validate_params(cls, params: SamplingParams):
|
||||||
|
|||||||
@@ -56,8 +56,7 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
class DummyPerReqLogitsProcessor:
|
class DummyPerReqLogitsProcessor:
|
||||||
"""The request-level logits processor masks out all logits except the
|
"""The request-level logits processor masks out all logits except the
|
||||||
token id identified by `target_token`
|
token id identified by `target_token`"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, target_token: int) -> None:
|
def __init__(self, target_token: int) -> None:
|
||||||
"""Specify `target_token`."""
|
"""Specify `target_token`."""
|
||||||
@@ -76,8 +75,7 @@ class DummyPerReqLogitsProcessor:
|
|||||||
|
|
||||||
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||||
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
||||||
info about the device type
|
info about the device type"""
|
||||||
"""
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def validate_params(cls, params: SamplingParams):
|
def validate_params(cls, params: SamplingParams):
|
||||||
|
|||||||
@@ -101,8 +101,7 @@ PROMPTS = ["The capital of France is", "The future of AI is"]
|
|||||||
class FSDPTrainWorker:
|
class FSDPTrainWorker:
|
||||||
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
|
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
|
||||||
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
|
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
|
||||||
actor with no producer mixin.
|
actor with no producer mixin."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, model_name, rank, world_size, master_addr, master_port):
|
def __init__(self, model_name, rank, world_size, master_addr, master_port):
|
||||||
self.rank = rank
|
self.rank = rank
|
||||||
|
|||||||
+3
-1
@@ -90,7 +90,9 @@ ignore = [
|
|||||||
# don't enforce that everything has a docstring
|
# don't enforce that everything has a docstring
|
||||||
"D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107",
|
"D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107",
|
||||||
# unwanted pydocstyle rules
|
# unwanted pydocstyle rules
|
||||||
"D205", "D301", "D400", "D401", "D404", "D415",
|
"D205", "D209", "D301", "D400", "D401", "D404", "D415",
|
||||||
|
# incompatible pydocstyle pairs; pin the half `ruff` already picks
|
||||||
|
"D203", "D213",
|
||||||
# star imports
|
# star imports
|
||||||
"F405", "F403",
|
"F405", "F403",
|
||||||
# lambda expression assignment
|
# lambda expression assignment
|
||||||
|
|||||||
@@ -724,8 +724,7 @@ class precompiled_wheel_utils:
|
|||||||
wheels: list[dict], repo_url: str, arch: str
|
wheels: list[dict], repo_url: str, arch: str
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Warn if installed torch differs from the custom ROCm build on
|
"""Warn if installed torch differs from the custom ROCm build on
|
||||||
wheels.vllm.ai and suggest the correct install command.
|
wheels.vllm.ai and suggest the correct install command."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
installed = torch.__version__
|
installed = torch.__version__
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -112,8 +112,7 @@ def test_basic_cumem():
|
|||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
def test_discard_tags():
|
def test_discard_tags():
|
||||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||||
tags while keeping other tags mapped and usable.
|
tags while keeping other tags mapped and usable."""
|
||||||
"""
|
|
||||||
allocator = get_mem_allocator_instance()
|
allocator = get_mem_allocator_instance()
|
||||||
|
|
||||||
with allocator.use_memory_pool("weights"):
|
with allocator.use_memory_pool("weights"):
|
||||||
|
|||||||
@@ -13,8 +13,7 @@ from vllm.benchmarks.datasets import BFCLDataset, get_samples
|
|||||||
|
|
||||||
def _patch_hf_api(side_effect):
|
def _patch_hf_api(side_effect):
|
||||||
"""Return a patch context that swaps `hf_api()` to a stub whose
|
"""Return a patch context that swaps `hf_api()` to a stub whose
|
||||||
`.hf_hub_download` attribute uses `side_effect`.
|
`.hf_hub_download` attribute uses `side_effect`."""
|
||||||
"""
|
|
||||||
fake_api = MagicMock()
|
fake_api = MagicMock()
|
||||||
fake_api.hf_hub_download.side_effect = side_effect
|
fake_api.hf_hub_download.side_effect = side_effect
|
||||||
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
|
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
|
||||||
@@ -116,8 +115,7 @@ def test_bfcl_dataset_translates_schema_and_attaches_tools(
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""BFCLDataset should translate schemas to OpenAI tool format, set
|
"""BFCLDataset should translate schemas to OpenAI tool format, set
|
||||||
`messages` directly on SampleRequest, and attach tools/tool_choice via
|
`messages` directly on SampleRequest, and attach tools/tool_choice via
|
||||||
request_overrides.
|
request_overrides."""
|
||||||
"""
|
|
||||||
paths = _write_fake_files(tmp_path)
|
paths = _write_fake_files(tmp_path)
|
||||||
|
|
||||||
def fake_download(_repo, filename, **_kwargs):
|
def fake_download(_repo, filename, **_kwargs):
|
||||||
@@ -168,8 +166,7 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
|
|||||||
hf_tokenizer: PreTrainedTokenizerBase,
|
hf_tokenizer: PreTrainedTokenizerBase,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""A typo'd category should produce an actionable ValueError, not an
|
"""A typo'd category should produce an actionable ValueError, not an
|
||||||
opaque huggingface_hub exception.
|
opaque huggingface_hub exception."""
|
||||||
"""
|
|
||||||
from huggingface_hub.errors import EntryNotFoundError
|
from huggingface_hub.errors import EntryNotFoundError
|
||||||
|
|
||||||
args = _args_for_bfcl(categories=["simpl"]) # typo
|
args = _args_for_bfcl(categories=["simpl"]) # typo
|
||||||
@@ -187,8 +184,7 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
|
|||||||
@pytest.mark.benchmark
|
@pytest.mark.benchmark
|
||||||
def test_chat_backend_uses_messages_field_when_set() -> None:
|
def test_chat_backend_uses_messages_field_when_set() -> None:
|
||||||
"""When RequestFuncInput.chat_messages is set, the chat backend must use
|
"""When RequestFuncInput.chat_messages is set, the chat backend must use
|
||||||
it verbatim and skip default content construction from `prompt`.
|
it verbatim and skip default content construction from `prompt`."""
|
||||||
"""
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||||
@@ -248,8 +244,7 @@ def test_chat_backend_uses_messages_field_when_set() -> None:
|
|||||||
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
||||||
"""prompt_len must reflect tokens from both messages *and* tool schemas,
|
"""prompt_len must reflect tokens from both messages *and* tool schemas,
|
||||||
so percentile buckets and input-distribution summaries aren't biased
|
so percentile buckets and input-distribution summaries aren't biased
|
||||||
low for tool-heavy traffic.
|
low for tool-heavy traffic."""
|
||||||
"""
|
|
||||||
paths = _write_fake_files(tmp_path)
|
paths = _write_fake_files(tmp_path)
|
||||||
|
|
||||||
def fake_download(_repo, filename, **_kwargs):
|
def fake_download(_repo, filename, **_kwargs):
|
||||||
@@ -296,8 +291,7 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
|
|||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Older tokenizers don't accept tools=; fallback must still produce a
|
"""Older tokenizers don't accept tools=; fallback must still produce a
|
||||||
non-zero prompt_len without crashing.
|
non-zero prompt_len without crashing."""
|
||||||
"""
|
|
||||||
paths = _write_fake_files(tmp_path)
|
paths = _write_fake_files(tmp_path)
|
||||||
|
|
||||||
def fake_download(_repo, filename, **_kwargs):
|
def fake_download(_repo, filename, **_kwargs):
|
||||||
|
|||||||
@@ -12,8 +12,7 @@ from vllm.benchmarks.datasets import get_samples
|
|||||||
class _RecordingTokenizer:
|
class _RecordingTokenizer:
|
||||||
"""Minimal tokenizer stub that records the kwargs forwarded to
|
"""Minimal tokenizer stub that records the kwargs forwarded to
|
||||||
apply_chat_template, so we can assert chat_template_kwargs propagation
|
apply_chat_template, so we can assert chat_template_kwargs propagation
|
||||||
without loading a real model/template.
|
without loading a real model/template."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.captured_kwargs: dict | None = None
|
self.captured_kwargs: dict | None = None
|
||||||
|
|||||||
@@ -10,8 +10,7 @@ from vllm.tokenizers import TokenizerLike
|
|||||||
|
|
||||||
class _FakeTokenizer(TokenizerLike):
|
class _FakeTokenizer(TokenizerLike):
|
||||||
"""Minimal tokenizer implementing the TokenizerLike protocol
|
"""Minimal tokenizer implementing the TokenizerLike protocol
|
||||||
for testing get_sampling_params.
|
for testing get_sampling_params."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
|
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
|
||||||
self._vocab_size = vocab_size
|
self._vocab_size = vocab_size
|
||||||
@@ -172,8 +171,7 @@ class TestGetSamplingParams:
|
|||||||
|
|
||||||
def test_output_low_clamped_to_one(self):
|
def test_output_low_clamped_to_one(self):
|
||||||
"""Even with a high ratio that would push output_low to 0,
|
"""Even with a high ratio that would push output_low to 0,
|
||||||
the function clamps it to 1.
|
the function clamps it to 1."""
|
||||||
"""
|
|
||||||
rng = np.random.default_rng(0)
|
rng = np.random.default_rng(0)
|
||||||
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
|
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
|
||||||
_, output_lens, _ = get_sampling_params(
|
_, output_lens, _ = get_sampling_params(
|
||||||
@@ -235,8 +233,7 @@ class TestGetSamplingParams:
|
|||||||
|
|
||||||
def test_input_len_zero_with_special_tokens(self):
|
def test_input_len_zero_with_special_tokens(self):
|
||||||
"""input_len < num_special_tokens → real_input_len = 0, which is fine
|
"""input_len < num_special_tokens → real_input_len = 0, which is fine
|
||||||
(range [0, 0]).
|
(range [0, 0])."""
|
||||||
"""
|
|
||||||
rng = np.random.default_rng(0)
|
rng = np.random.default_rng(0)
|
||||||
input_lens, _, _ = get_sampling_params(
|
input_lens, _, _ = get_sampling_params(
|
||||||
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
|
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
|
||||||
|
|||||||
@@ -52,8 +52,7 @@ def _write_dataset(path: Path) -> None:
|
|||||||
|
|
||||||
def _args(dataset_path: str) -> argparse.Namespace:
|
def _args(dataset_path: str) -> argparse.Namespace:
|
||||||
"""Reproduce the argparse.Namespace that serve.py builds from the
|
"""Reproduce the argparse.Namespace that serve.py builds from the
|
||||||
failing command, including skip_tokenizer_init=True.
|
failing command, including skip_tokenizer_init=True."""
|
||||||
"""
|
|
||||||
return argparse.Namespace(
|
return argparse.Namespace(
|
||||||
# dataset
|
# dataset
|
||||||
dataset_name="custom",
|
dataset_name="custom",
|
||||||
|
|||||||
@@ -732,8 +732,7 @@ def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
|
|||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
):
|
):
|
||||||
"""Verify that without GDN layers in static_forward_context,
|
"""Verify that without GDN layers in static_forward_context,
|
||||||
the gated pattern is not registered and no matches occur.
|
the gated pattern is not registered and no matches occur."""
|
||||||
"""
|
|
||||||
group_shape = GroupShape(1, 128)
|
group_shape = GroupShape(1, 128)
|
||||||
vllm_config = VllmConfig(
|
vllm_config = VllmConfig(
|
||||||
model_config=ModelConfig(dtype=dtype),
|
model_config=ModelConfig(dtype=dtype),
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ DEVICE_TYPE = current_platform.device_type
|
|||||||
|
|
||||||
class SplitCoalescingModel(torch.nn.Module):
|
class SplitCoalescingModel(torch.nn.Module):
|
||||||
"""Model with 3 separate split_with_sizes calls on the same input,
|
"""Model with 3 separate split_with_sizes calls on the same input,
|
||||||
simulating the B200+FP8 graph where CSE fails to merge them.
|
simulating the B200+FP8 graph where CSE fails to merge them."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, q_size: int, kv_size: int) -> None:
|
def __init__(self, q_size: int, kv_size: int) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|||||||
@@ -77,8 +77,7 @@ def test_non_primitive_kwargs_lifted_to_consts(
|
|||||||
into the namespace, so ``device`` is unbound. The fix collects such
|
into the namespace, so ``device`` is unbound. The fix collects such
|
||||||
objects into ``__vllm_consts__`` and references them by index. The
|
objects into ``__vllm_consts__`` and references them by index. The
|
||||||
unqualified ``device(type=...)`` form must never appear in the
|
unqualified ``device(type=...)`` form must never appear in the
|
||||||
generated source.
|
generated source."""
|
||||||
"""
|
|
||||||
split_gm = _trace_and_split(model_fn, (x,), split_ops)
|
split_gm = _trace_and_split(model_fn, (x,), split_ops)
|
||||||
code, submod_names, consts = generate_execution_code(split_gm)
|
code, submod_names, consts = generate_execution_code(split_gm)
|
||||||
|
|
||||||
@@ -99,8 +98,7 @@ def test_dtype_singleton_deduped(x: torch.Tensor) -> None:
|
|||||||
"""``torch.float16`` is a process-wide singleton, so two ops referring
|
"""``torch.float16`` is a process-wide singleton, so two ops referring
|
||||||
to it in the traced graph share a single consts slot via ``id()``-based
|
to it in the traced graph share a single consts slot via ``id()``-based
|
||||||
dedup. Distinct expressions (``x.to(...)`` vs ``(x*2).to(...)``) ensure
|
dedup. Distinct expressions (``x.to(...)`` vs ``(x*2).to(...)``) ensure
|
||||||
the tracer can't CSE the two ops into a single node.
|
the tracer can't CSE the two ops into a single node."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
return x.to(dtype=torch.float16) + (x * 2).to(dtype=torch.float16)
|
return x.to(dtype=torch.float16) + (x * 2).to(dtype=torch.float16)
|
||||||
@@ -149,8 +147,7 @@ def test_distinct_dtypes_get_distinct_slots(x: torch.Tensor) -> None:
|
|||||||
def test_consts_ordering_deterministic(x: torch.Tensor) -> None:
|
def test_consts_ordering_deterministic(x: torch.Tensor) -> None:
|
||||||
"""Two independent traces of the same model must produce equal consts
|
"""Two independent traces of the same model must produce equal consts
|
||||||
lists *in the same order*. Cache artifacts identify const slots by
|
lists *in the same order*. Cache artifacts identify const slots by
|
||||||
index, so a non-deterministic order would invalidate cached code.
|
index, so a non-deterministic order would invalidate cached code."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
# Multiple distinct non-primitives encountered in a fixed graph order.
|
# Multiple distinct non-primitives encountered in a fixed graph order.
|
||||||
@@ -188,8 +185,7 @@ def test_consts_shared_across_split_submods(x: torch.Tensor) -> None:
|
|||||||
The function below splits into three inlined submods, two of which
|
The function below splits into three inlined submods, two of which
|
||||||
independently reference ``torch.float16``. The shared ``const_index``
|
independently reference ``torch.float16``. The shared ``const_index``
|
||||||
threaded through recursive ``generate_execution_code_with_name`` calls
|
threaded through recursive ``generate_execution_code_with_name`` calls
|
||||||
must collapse the dtype to a single slot used from both submods.
|
must collapse the dtype to a single slot used from both submods."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
a = x.to(dtype=torch.float16) # submod_0: _to_copy(fp16)
|
a = x.to(dtype=torch.float16) # submod_0: _to_copy(fp16)
|
||||||
@@ -234,8 +230,7 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
|
|||||||
"""When a child of split_gm is *not* a ``torch.fx.GraphModule`` — as
|
"""When a child of split_gm is *not* a ``torch.fx.GraphModule`` — as
|
||||||
happens in production once ``PiecewiseBackend`` replaces submods —
|
happens in production once ``PiecewiseBackend`` replaces submods —
|
||||||
codegen emits ``__vllm_submods__[idx](...)`` instead of inlining, and
|
codegen emits ``__vllm_submods__[idx](...)`` instead of inlining, and
|
||||||
the runtime callable is bound from ``submod_callables``.
|
the runtime callable is bound from ``submod_callables``."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
return x.relu().sigmoid()
|
return x.relu().sigmoid()
|
||||||
@@ -285,8 +280,7 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
|
|||||||
def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
|
def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
|
||||||
"""``operator.getitem`` on submod tuple returns is the ``call_function``
|
"""``operator.getitem`` on submod tuple returns is the ``call_function``
|
||||||
special case at codegen.py — emitted as ``name = source[index]``
|
special case at codegen.py — emitted as ``name = source[index]``
|
||||||
rather than a function call.
|
rather than a function call."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
return x.relu().sigmoid()
|
return x.relu().sigmoid()
|
||||||
@@ -304,8 +298,7 @@ def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
|
|||||||
def test_del_emitted_for_intermediate_values(x: torch.Tensor) -> None:
|
def test_del_emitted_for_intermediate_values(x: torch.Tensor) -> None:
|
||||||
"""The codegen schedules ``del`` after a value's last use to free
|
"""The codegen schedules ``del`` after a value's last use to free
|
||||||
memory early. Multi-submod splits naturally have intermediates whose
|
memory early. Multi-submod splits naturally have intermediates whose
|
||||||
last use is not the output node.
|
last use is not the output node."""
|
||||||
"""
|
|
||||||
|
|
||||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||||
return x.relu().sigmoid().tanh()
|
return x.relu().sigmoid().tanh()
|
||||||
@@ -325,8 +318,7 @@ def test_with_submod_false_rejects_call_module() -> None:
|
|||||||
"""``generate_execution_code_with_name(with_submod=False)`` is the
|
"""``generate_execution_code_with_name(with_submod=False)`` is the
|
||||||
recursive entry for inlining a GraphModule into its parent. It must
|
recursive entry for inlining a GraphModule into its parent. It must
|
||||||
refuse a graph that itself contains ``call_module`` nodes — the parent
|
refuse a graph that itself contains ``call_module`` nodes — the parent
|
||||||
is responsible for handling those.
|
is responsible for handling those."""
|
||||||
"""
|
|
||||||
g = fx.Graph()
|
g = fx.Graph()
|
||||||
x_node = g.placeholder("x")
|
x_node = g.placeholder("x")
|
||||||
root = torch.nn.Module()
|
root = torch.nn.Module()
|
||||||
@@ -343,8 +335,7 @@ def test_node_ref_recurses_through_containers() -> None:
|
|||||||
"""``_node_ref`` is the recursive walker that lifts non-primitives
|
"""``_node_ref`` is the recursive walker that lifts non-primitives
|
||||||
nested inside list/tuple/dict args. Real aten ops rarely produce such
|
nested inside list/tuple/dict args. Real aten ops rarely produce such
|
||||||
structures, but the path is needed for DTensor placement lists and
|
structures, but the path is needed for DTensor placement lists and
|
||||||
other future cases — unit-test the walker directly.
|
other future cases — unit-test the walker directly."""
|
||||||
"""
|
|
||||||
consts: list = []
|
consts: list = []
|
||||||
const_index: dict[int, int] = {}
|
const_index: dict[int, int] = {}
|
||||||
cpu = torch.device("cpu")
|
cpu = torch.device("cpu")
|
||||||
@@ -368,8 +359,7 @@ def test_node_ref_recurses_through_containers() -> None:
|
|||||||
def test_legacy_code_without_consts() -> None:
|
def test_legacy_code_without_consts() -> None:
|
||||||
"""``compile_execution_fn(consts=None)`` must still load code that has
|
"""``compile_execution_fn(consts=None)`` must still load code that has
|
||||||
no ``__vllm_consts__`` reference, so older serialized cache artifacts
|
no ``__vllm_consts__`` reference, so older serialized cache artifacts
|
||||||
keep working.
|
keep working."""
|
||||||
"""
|
|
||||||
# Pre-consts codegen: no __vllm_consts__ reference, only torch/operator.
|
# Pre-consts codegen: no __vllm_consts__ reference, only torch/operator.
|
||||||
legacy_code = (
|
legacy_code = (
|
||||||
"import torch\n"
|
"import torch\n"
|
||||||
|
|||||||
@@ -129,8 +129,7 @@ def test_compile_config_get_compile_ranges():
|
|||||||
|
|
||||||
class PostGradStaticShapeChecker(InductorPass):
|
class PostGradStaticShapeChecker(InductorPass):
|
||||||
"""Asserts that compile_sizes entries produce graphs with fully concrete
|
"""Asserts that compile_sizes entries produce graphs with fully concrete
|
||||||
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes.
|
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.num_static_calls = 0
|
self.num_static_calls = 0
|
||||||
@@ -171,8 +170,7 @@ class PostGradStaticShapeChecker(InductorPass):
|
|||||||
|
|
||||||
def test_compile_sizes_produce_static_shapes(disable_vllm_compile_cache):
|
def test_compile_sizes_produce_static_shapes(disable_vllm_compile_cache):
|
||||||
"""Verify that compile_sizes entries are compiled with fully concrete
|
"""Verify that compile_sizes entries are compiled with fully concrete
|
||||||
shapes (no SymInts), while compile_ranges entries retain dynamic shapes.
|
shapes (no SymInts), while compile_ranges entries retain dynamic shapes."""
|
||||||
"""
|
|
||||||
checker = PostGradStaticShapeChecker()
|
checker = PostGradStaticShapeChecker()
|
||||||
torch.set_default_device("cuda")
|
torch.set_default_device("cuda")
|
||||||
vllm_config = VllmConfig(
|
vllm_config = VllmConfig(
|
||||||
|
|||||||
@@ -1229,8 +1229,7 @@ def test_compile_sizes_padding_validation():
|
|||||||
|
|
||||||
def test_inductor_asserts_default_disabled(monkeypatch):
|
def test_inductor_asserts_default_disabled(monkeypatch):
|
||||||
"""Test that inductor runtime asserts are disabled by default
|
"""Test that inductor runtime asserts are disabled by default
|
||||||
(INFO logging level) on torch < 2.12.
|
(INFO logging level) on torch < 2.12."""
|
||||||
"""
|
|
||||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
@@ -1248,8 +1247,7 @@ def test_inductor_asserts_default_disabled(monkeypatch):
|
|||||||
|
|
||||||
def test_inductor_asserts_enabled_in_debug(monkeypatch):
|
def test_inductor_asserts_enabled_in_debug(monkeypatch):
|
||||||
"""Test that VLLM_LOGGING_LEVEL=DEBUG enables inductor runtime asserts
|
"""Test that VLLM_LOGGING_LEVEL=DEBUG enables inductor runtime asserts
|
||||||
on torch < 2.12.
|
on torch < 2.12."""
|
||||||
"""
|
|
||||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "DEBUG")
|
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "DEBUG")
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
@@ -1285,8 +1283,7 @@ def test_get_inductor_factors_includes_configs():
|
|||||||
|
|
||||||
def test_inductor_asserts_user_override(monkeypatch):
|
def test_inductor_asserts_user_override(monkeypatch):
|
||||||
"""Test that explicit inductor_compile_config overrides the
|
"""Test that explicit inductor_compile_config overrides the
|
||||||
debug-logging default.
|
debug-logging default."""
|
||||||
"""
|
|
||||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
|
|||||||
@@ -236,8 +236,7 @@ class TestInductorFallbackFastPath:
|
|||||||
|
|
||||||
def test_inner_set_membership_still_works_for_standard_ops(self):
|
def test_inner_set_membership_still_works_for_standard_ops(self):
|
||||||
"""The patch must not break Inductor's existing fallback decisions
|
"""The patch must not break Inductor's existing fallback decisions
|
||||||
for non-vllm ops such as ``torchvision::roi_align``.
|
for non-vllm ops such as ``torchvision::roi_align``."""
|
||||||
"""
|
|
||||||
import torch._inductor.lowering as _lowering
|
import torch._inductor.lowering as _lowering
|
||||||
|
|
||||||
allow_list = _lowering.FALLBACK_ALLOW_LIST
|
allow_list = _lowering.FALLBACK_ALLOW_LIST
|
||||||
|
|||||||
@@ -372,8 +372,7 @@ class _HeterogeneousConfig(PretrainedConfig):
|
|||||||
"""A heterogeneous config with no convertor of its own.
|
"""A heterogeneous config with no convertor of its own.
|
||||||
|
|
||||||
Mirrors the parts vLLM uses: per-layer configs are shallow copies with the
|
Mirrors the parts vLLM uses: per-layer configs are shallow copies with the
|
||||||
varying attributes applied and heterogeneity stripped, so they do not recurse.
|
varying attributes applied and heterogeneity stripped, so they do not recurse."""
|
||||||
"""
|
|
||||||
|
|
||||||
is_heterogeneous = True
|
is_heterogeneous = True
|
||||||
|
|
||||||
@@ -472,8 +471,7 @@ def test_deepseek_v4_convertor_splits_vision_architecture():
|
|||||||
"""The vision checkpoint shares model_type/architectures with the
|
"""The vision checkpoint shares model_type/architectures with the
|
||||||
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
|
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
|
||||||
by rewriting hf_config.architectures (model-class resolution reads the
|
by rewriting hf_config.architectures (model-class resolution reads the
|
||||||
raw attribute).
|
raw attribute)."""
|
||||||
"""
|
|
||||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||||
DeepseekV4ModelArchConfigConvertor,
|
DeepseekV4ModelArchConfigConvertor,
|
||||||
|
|||||||
@@ -42,8 +42,7 @@ def test_mm_encoder_attn_backend_hash_updates():
|
|||||||
|
|
||||||
def test_language_model_only_does_not_affect_mm_hash():
|
def test_language_model_only_does_not_affect_mm_hash():
|
||||||
"""language_model_only does not affect the ViT computation graph,
|
"""language_model_only does not affect the ViT computation graph,
|
||||||
so it should not change the multimodal config hash.
|
so it should not change the multimodal config hash."""
|
||||||
"""
|
|
||||||
base_hash = MultiModalConfig().compute_hash()
|
base_hash = MultiModalConfig().compute_hash()
|
||||||
lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash()
|
lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash()
|
||||||
assert base_hash == lm_only_hash
|
assert base_hash == lm_only_hash
|
||||||
@@ -51,8 +50,7 @@ def test_language_model_only_does_not_affect_mm_hash():
|
|||||||
|
|
||||||
def test_language_model_only_affects_model_hash():
|
def test_language_model_only_affects_model_hash():
|
||||||
"""language_model_only affects the LM computation graph,
|
"""language_model_only affects the LM computation graph,
|
||||||
so it should change the model config hash.
|
so it should change the model config hash."""
|
||||||
"""
|
|
||||||
model = "llava-hf/llava-1.5-7b-hf"
|
model = "llava-hf/llava-1.5-7b-hf"
|
||||||
base_hash = ModelConfig(model).compute_hash()
|
base_hash = ModelConfig(model).compute_hash()
|
||||||
lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash()
|
lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash()
|
||||||
@@ -388,8 +386,7 @@ def _resolve_mm_video_decode_device(
|
|||||||
torchcodec_available: bool = True,
|
torchcodec_available: bool = True,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Run processor-device then video-decode-device resolution and report
|
"""Run processor-device then video-decode-device resolution and report
|
||||||
the resulting video media IO kwargs.
|
the resulting video media IO kwargs."""
|
||||||
"""
|
|
||||||
mm_config = MultiModalConfig(
|
mm_config = MultiModalConfig(
|
||||||
mm_processor_kwargs={} if device is None else {"device": device},
|
mm_processor_kwargs={} if device is None else {"device": device},
|
||||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||||
@@ -419,8 +416,7 @@ def _resolve_mm_video_decode_device(
|
|||||||
|
|
||||||
def test_auto_video_decode_uses_nvdec_on_encoder_instance():
|
def test_auto_video_decode_uses_nvdec_on_encoder_instance():
|
||||||
"""The processor runs on the accelerator there, so decoded frames should
|
"""The processor runs on the accelerator there, so decoded frames should
|
||||||
stay on-device too.
|
stay on-device too."""
|
||||||
"""
|
|
||||||
assert _resolve_mm_video_decode_device(ec_role="ec_producer") == {
|
assert _resolve_mm_video_decode_device(ec_role="ec_producer") == {
|
||||||
"backend": "torchcodec",
|
"backend": "torchcodec",
|
||||||
"device": "cuda",
|
"device": "cuda",
|
||||||
@@ -432,8 +428,7 @@ def test_auto_video_decode_stays_on_cpu_off_encoder_instance(
|
|||||||
ec_role: ECRole | None,
|
ec_role: ECRole | None,
|
||||||
):
|
):
|
||||||
"""The processor stays on CPU off encode-only instances, so video
|
"""The processor stays on CPU off encode-only instances, so video
|
||||||
decoding should too.
|
decoding should too."""
|
||||||
"""
|
|
||||||
assert _resolve_mm_video_decode_device(ec_role=ec_role) == {}
|
assert _resolve_mm_video_decode_device(ec_role=ec_role) == {}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -34,8 +34,7 @@ def _make_hf_config(**kwargs) -> PretrainedConfig:
|
|||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
def test_dict_overrides_are_not_forwarded_to_draft():
|
def test_dict_overrides_are_not_forwarded_to_draft():
|
||||||
"""Dict overrides are target-specific key patches; the draft must get
|
"""Dict overrides are target-specific key patches; the draft must get
|
||||||
only the architecture-mapping override.
|
only the architecture-mapping override."""
|
||||||
"""
|
|
||||||
composed = SpeculativeConfig.compose_draft_hf_overrides(
|
composed = SpeculativeConfig.compose_draft_hf_overrides(
|
||||||
{"max_position_embeddings": 1234}
|
{"max_position_embeddings": 1234}
|
||||||
)
|
)
|
||||||
@@ -51,8 +50,7 @@ def test_none_overrides_fall_back_to_arch_mapping():
|
|||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
def test_callable_overrides_reach_the_draft_config():
|
def test_callable_overrides_reach_the_draft_config():
|
||||||
"""A callable override (config-to-config transform) composes with the
|
"""A callable override (config-to-config transform) composes with the
|
||||||
architecture-mapping override and is applied to the draft config.
|
architecture-mapping override and is applied to the draft config."""
|
||||||
"""
|
|
||||||
|
|
||||||
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
|
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||||
hf_config.num_hidden_layers = 1
|
hf_config.num_hidden_layers = 1
|
||||||
@@ -69,8 +67,7 @@ def test_callable_overrides_reach_the_draft_config():
|
|||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
def test_arch_mapping_applies_before_callable_override():
|
def test_arch_mapping_applies_before_callable_override():
|
||||||
"""The static arch-mapping override runs first, so the user callable
|
"""The static arch-mapping override runs first, so the user callable
|
||||||
observes (and may adjust) the post-mapping config.
|
observes (and may adjust) the post-mapping config."""
|
||||||
"""
|
|
||||||
seen_architectures: list[str] = []
|
seen_architectures: list[str] = []
|
||||||
|
|
||||||
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
|
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||||
@@ -131,8 +128,7 @@ def test_composed_override_is_picklable():
|
|||||||
the composed override must be picklable. A nested local closure is not
|
the composed override must be picklable. A nested local closure is not
|
||||||
(it raised ``Can't get local object`` on DFlashDraftModel); a
|
(it raised ``Can't get local object`` on DFlashDraftModel); a
|
||||||
``functools.partial`` over a module-referenceable static method is.
|
``functools.partial`` over a module-referenceable static method is.
|
||||||
Guard against regressing to a closure.
|
Guard against regressing to a closure."""
|
||||||
"""
|
|
||||||
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
|
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
|
||||||
|
|
||||||
assert isinstance(composed, functools.partial)
|
assert isinstance(composed, functools.partial)
|
||||||
|
|||||||
@@ -99,8 +99,7 @@ def test_independent_draft_model_keeps_its_own_limit(
|
|||||||
vllm_caplog: pytest.LogCaptureFixture,
|
vllm_caplog: pytest.LogCaptureFixture,
|
||||||
):
|
):
|
||||||
"""An independent AR draft may genuinely have a smaller context than the
|
"""An independent AR draft may genuinely have a smaller context than the
|
||||||
target; its max_position_embeddings must not be resized.
|
target; its max_position_embeddings must not be resized."""
|
||||||
"""
|
|
||||||
target_model_config = ModelConfig(
|
target_model_config = ModelConfig(
|
||||||
AR_MODEL, hf_overrides={"max_position_embeddings": 8192}
|
AR_MODEL, hf_overrides={"max_position_embeddings": 8192}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -179,8 +179,7 @@ class MockSubscriber:
|
|||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def enable_ray_v2_backend():
|
def enable_ray_v2_backend():
|
||||||
"""Set env vars for the Ray V2 executor backend and shut down Ray
|
"""Set env vars for the Ray V2 executor backend and shut down Ray
|
||||||
between tests.
|
between tests."""
|
||||||
"""
|
|
||||||
import ray
|
import ray
|
||||||
|
|
||||||
saved = {
|
saved = {
|
||||||
|
|||||||
@@ -124,8 +124,7 @@ def _build_processed_layer(
|
|||||||
backend: str, raw: dict[str, torch.Tensor], device: torch.device
|
backend: str, raw: dict[str, torch.Tensor], device: torch.device
|
||||||
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
|
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
|
||||||
"""Create the method + layer, load ``raw`` and run the real
|
"""Create the method + layer, load ``raw`` and run the real
|
||||||
process_weights_after_loading (which builds the flashinfer kernel).
|
process_weights_after_loading (which builds the flashinfer kernel)."""
|
||||||
"""
|
|
||||||
moe_config = _make_moe_config(backend)
|
moe_config = _make_moe_config(backend)
|
||||||
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
|
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
|
||||||
layer = _RoutedExpertsStub(moe_config).to(device)
|
layer = _RoutedExpertsStub(moe_config).to(device)
|
||||||
@@ -158,8 +157,7 @@ def _assert_tensors_equal(
|
|||||||
|
|
||||||
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
|
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
|
||||||
"""Permute experts of every registered Parameter in place, the way
|
"""Permute experts of every registered Parameter in place, the way
|
||||||
EPLB's in-place rearrangement moves expert slices along dim 0.
|
EPLB's in-place rearrangement moves expert slices along dim 0."""
|
||||||
"""
|
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
for _, param in layer.named_parameters():
|
for _, param in layer.named_parameters():
|
||||||
param.copy_(param[perm])
|
param.copy_(param[perm])
|
||||||
@@ -167,8 +165,7 @@ def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) ->
|
|||||||
|
|
||||||
def _ensure_world1_distributed() -> None:
|
def _ensure_world1_distributed() -> None:
|
||||||
"""The enable_eplb config makes weight processing all-reduce activation
|
"""The enable_eplb config makes weight processing all-reduce activation
|
||||||
scale amaxes over the EP group, so a (single-rank) EP group must exist.
|
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
|
||||||
"""
|
|
||||||
from vllm.distributed.parallel_state import (
|
from vllm.distributed.parallel_state import (
|
||||||
ensure_model_parallel_initialized,
|
ensure_model_parallel_initialized,
|
||||||
init_distributed_environment,
|
init_distributed_environment,
|
||||||
@@ -256,8 +253,7 @@ def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
|
|||||||
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
|
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
|
||||||
"""CPU-only: the fp8 per-tensor oracle must register the fused
|
"""CPU-only: the fp8 per-tensor oracle must register the fused
|
||||||
(w_scale * a_scale) products as layer Parameters aliased by the quant
|
(w_scale * a_scale) products as layer Parameters aliased by the quant
|
||||||
config, so EPLB rearrangement keeps kernels consistent.
|
config, so EPLB rearrangement keeps kernels consistent."""
|
||||||
"""
|
|
||||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||||
Fp8MoeBackend,
|
Fp8MoeBackend,
|
||||||
make_fp8_moe_quant_config,
|
make_fp8_moe_quant_config,
|
||||||
|
|||||||
@@ -160,8 +160,7 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
|||||||
class _AttnMeta:
|
class _AttnMeta:
|
||||||
"""Minimal placeholder so set_forward_context's
|
"""Minimal placeholder so set_forward_context's
|
||||||
``attn_metadata is not None`` guard (forward_context.py:334)
|
``attn_metadata is not None`` guard (forward_context.py:334)
|
||||||
is satisfied. The real DPMetadata is built from num_tokens_across_dp.
|
is satisfied. The real DPMetadata is built from num_tokens_across_dp."""
|
||||||
"""
|
|
||||||
|
|
||||||
dp_metadata = None
|
dp_metadata = None
|
||||||
|
|
||||||
@@ -460,8 +459,7 @@ def _one_sided_workspace_grow_worker(rank, world_size):
|
|||||||
def test_one_sided_manager_workspace_grow(world_size):
|
def test_one_sided_manager_workspace_grow(world_size):
|
||||||
"""A later initialize() with a larger per-token payload must grow the
|
"""A later initialize() with a larger per-token payload must grow the
|
||||||
workspace and rebuild MoeAlltoAll; a later initialize() with a smaller
|
workspace and rebuild MoeAlltoAll; a later initialize() with a smaller
|
||||||
payload must no-op.
|
payload must no-op."""
|
||||||
"""
|
|
||||||
_spawn_workers(
|
_spawn_workers(
|
||||||
_one_sided_workspace_grow_worker,
|
_one_sided_workspace_grow_worker,
|
||||||
world_size,
|
world_size,
|
||||||
|
|||||||
@@ -21,8 +21,7 @@ from vllm.utils.network_utils import get_ip
|
|||||||
|
|
||||||
def test_mq_bind_with_local_ip():
|
def test_mq_bind_with_local_ip():
|
||||||
"""MessageQueue with remote readers should successfully bind
|
"""MessageQueue with remote readers should successfully bind
|
||||||
when connect_ip is the local node's IP.
|
when connect_ip is the local node's IP."""
|
||||||
"""
|
|
||||||
# n_reader=2, n_local_reader=1 means 1 remote reader,
|
# n_reader=2, n_local_reader=1 means 1 remote reader,
|
||||||
# which triggers the remote ZMQ socket bind.
|
# which triggers the remote ZMQ socket bind.
|
||||||
mq = MessageQueue(
|
mq = MessageQueue(
|
||||||
@@ -44,8 +43,7 @@ def test_mq_bind_with_local_ip():
|
|||||||
def test_mq_bind_with_non_local_ip_fails():
|
def test_mq_bind_with_non_local_ip_fails():
|
||||||
"""MessageQueue should fail to bind when connect_ip is a
|
"""MessageQueue should fail to bind when connect_ip is a
|
||||||
non-local IP address (simulating the bug where master_addr
|
non-local IP address (simulating the bug where master_addr
|
||||||
from a different node was used).
|
from a different node was used)."""
|
||||||
"""
|
|
||||||
# Use a non-local IP that we definitely can't bind to.
|
# Use a non-local IP that we definitely can't bind to.
|
||||||
# 198.51.100.1 is from TEST-NET-2 (RFC 5737), never locally assigned.
|
# 198.51.100.1 is from TEST-NET-2 (RFC 5737), never locally assigned.
|
||||||
non_local_ip = "198.51.100.1"
|
non_local_ip = "198.51.100.1"
|
||||||
@@ -59,8 +57,7 @@ def test_mq_bind_with_non_local_ip_fails():
|
|||||||
|
|
||||||
def test_mq_bind_defaults_to_local_ip():
|
def test_mq_bind_defaults_to_local_ip():
|
||||||
"""When connect_ip is None, MessageQueue should auto-detect
|
"""When connect_ip is None, MessageQueue should auto-detect
|
||||||
the local IP and bind successfully.
|
the local IP and bind successfully."""
|
||||||
"""
|
|
||||||
mq = MessageQueue(
|
mq = MessageQueue(
|
||||||
n_reader=2,
|
n_reader=2,
|
||||||
n_local_reader=1,
|
n_local_reader=1,
|
||||||
|
|||||||
@@ -495,8 +495,7 @@ def test_pynccl_suspend_resume_idempotent():
|
|||||||
|
|
||||||
def test_pynccl_suspend_noop_without_symbol():
|
def test_pynccl_suspend_noop_without_symbol():
|
||||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||||
crash, and resume() must then no-op too since nothing was suspended.
|
crash, and resume() must then no-op too since nothing was suspended."""
|
||||||
"""
|
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
|
|
||||||
comm = object.__new__(PyNcclCommunicator)
|
comm = object.__new__(PyNcclCommunicator)
|
||||||
|
|||||||
@@ -23,8 +23,7 @@ def _ray_init():
|
|||||||
|
|
||||||
Without this, workers cannot unpickle actor classes defined in the
|
Without this, workers cannot unpickle actor classes defined in the
|
||||||
``tests`` package, causing FunctionActorManager to fall back to
|
``tests`` package, causing FunctionActorManager to fall back to
|
||||||
TemporaryActor which drops async method signatures.
|
TemporaryActor which drops async method signatures."""
|
||||||
"""
|
|
||||||
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
||||||
ray.init(
|
ray.init(
|
||||||
ignore_reinit_error=True,
|
ignore_reinit_error=True,
|
||||||
|
|||||||
@@ -53,8 +53,7 @@ META = torch.device("meta")
|
|||||||
|
|
||||||
class _FakeLayer:
|
class _FakeLayer:
|
||||||
"""Stand-in for a leaf module. The planner only stores it and keys on
|
"""Stand-in for a leaf module. The planner only stores it and keys on
|
||||||
``id()``; nothing in the plan path touches its attributes.
|
``id()``; nothing in the plan path touches its attributes."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, tag: str) -> None:
|
def __init__(self, tag: str) -> None:
|
||||||
self.tag = tag
|
self.tag = tag
|
||||||
@@ -65,8 +64,7 @@ class _FakeLayer:
|
|||||||
|
|
||||||
def _module(layer, copies):
|
def _module(layer, copies):
|
||||||
"""What the bake records for one leaf module: a scatter list whose entries
|
"""What the bake records for one leaf module: a scatter list whose entries
|
||||||
all carry that module as their ``layer``.
|
all carry that module as their ``layer``."""
|
||||||
"""
|
|
||||||
for c in copies:
|
for c in copies:
|
||||||
c.layer = layer
|
c.layer = layer
|
||||||
return copies
|
return copies
|
||||||
@@ -83,8 +81,7 @@ def _copy(
|
|||||||
dtype=torch.bfloat16,
|
dtype=torch.bfloat16,
|
||||||
):
|
):
|
||||||
"""A `_Scatter` as the bake would record it: source key, owning layer, and
|
"""A `_Scatter` as the bake would record it: source key, owning layer, and
|
||||||
the meta destination region.
|
the meta destination region."""
|
||||||
"""
|
|
||||||
stride = (
|
stride = (
|
||||||
(1,)
|
(1,)
|
||||||
if len(shape) == 1
|
if len(shape) == 1
|
||||||
@@ -143,8 +140,7 @@ def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None):
|
|||||||
|
|
||||||
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
||||||
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
|
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
|
||||||
where every gather group is one fully-baked leaf module.
|
where every gather group is one fully-baked leaf module."""
|
||||||
"""
|
|
||||||
names = ["embed.weight"]
|
names = ["embed.weight"]
|
||||||
names += [f"model.layers.{i}.w" for i in range(n_layers)]
|
names += [f"model.layers.{i}.w" for i in range(n_layers)]
|
||||||
names += ["norm.weight"]
|
names += ["norm.weight"]
|
||||||
@@ -165,8 +161,7 @@ def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
|||||||
class TestFakeOpChains:
|
class TestFakeOpChains:
|
||||||
"""Every allowlisted op must append itself to the chain and hand back a
|
"""Every allowlisted op must append itself to the chain and hand back a
|
||||||
child whose shape/dtype PyTorch itself computed. The chain is the wire
|
child whose shape/dtype PyTorch itself computed. The chain is the wire
|
||||||
format the producer replays, so its exact contents are load-bearing.
|
format the producer replays, so its exact contents are load-bearing."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
|
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
|
||||||
return FakeRDTTensor(
|
return FakeRDTTensor(
|
||||||
@@ -227,8 +222,7 @@ class TestFakeOpChains:
|
|||||||
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
|
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
|
||||||
def test_multi_return_op_emits_one_child_per_output(self, op, n):
|
def test_multi_return_op_emits_one_child_per_output(self, op, n):
|
||||||
"""chunk/unbind hand back a tuple; each child carries the base op plus a
|
"""chunk/unbind hand back a tuple; each child carries the base op plus a
|
||||||
trailing __getitem__(i) so the producer can index the replayed result.
|
trailing __getitem__(i) so the producer can index the replayed result."""
|
||||||
"""
|
|
||||||
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
|
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
|
||||||
assert isinstance(parts, tuple)
|
assert isinstance(parts, tuple)
|
||||||
assert len(parts) == n
|
assert len(parts) == n
|
||||||
@@ -244,8 +238,7 @@ class TestFakeOpChains:
|
|||||||
|
|
||||||
class TestFakeUnsupportedOps:
|
class TestFakeUnsupportedOps:
|
||||||
"""Anything that needs real data must fail loudly at bake time rather than
|
"""Anything that needs real data must fail loudly at bake time rather than
|
||||||
silently transferring the wrong bytes.
|
silently transferring the wrong bytes."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _fake(self):
|
def _fake(self):
|
||||||
return FakeRDTTensor(
|
return FakeRDTTensor(
|
||||||
@@ -285,8 +278,7 @@ class TestFakeUnsupportedOps:
|
|||||||
|
|
||||||
class TestBakeRecording:
|
class TestBakeRecording:
|
||||||
"""During the dry run the fake's ``copy_`` is the data sink: it records the
|
"""During the dry run the fake's ``copy_`` is the data sink: it records the
|
||||||
source chain plus the meta destination's strided region and moves nothing.
|
source chain plus the meta destination's strided region and moves nothing."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _recorder_and_fake(self, shape=(4, 6)):
|
def _recorder_and_fake(self, shape=(4, 6)):
|
||||||
rec = BakeSink()
|
rec = BakeSink()
|
||||||
@@ -322,8 +314,7 @@ class TestBakeRecording:
|
|||||||
|
|
||||||
def test_unattributed_copy_is_live_but_unrecorded(self):
|
def test_unattributed_copy_is_live_but_unrecorded(self):
|
||||||
"""A copy_ with no loader stamp cannot be attributed to a param, so its
|
"""A copy_ with no loader stamp cannot be attributed to a param, so its
|
||||||
module must fall back to the plain load — but the name still moved data.
|
module must fall back to the plain load — but the name still moved data."""
|
||||||
"""
|
|
||||||
rec, fake = self._recorder_and_fake()
|
rec, fake = self._recorder_and_fake()
|
||||||
rec.current = None
|
rec.current = None
|
||||||
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
|
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
|
||||||
@@ -363,8 +354,7 @@ class TestBakeRecording:
|
|||||||
producer packs what the replay yields, so the record must carry the
|
producer packs what the replay yields, so the record must carry the
|
||||||
POST-chain dtype: taking it from the source name's metadata instead sizes
|
POST-chain dtype: taking it from the source name's metadata instead sizes
|
||||||
the slice with the wrong itemsize and shifts every later slice in the
|
the slice with the wrong itemsize and shifts every later slice in the
|
||||||
chunk, carving the packed blob differently on the two sides.
|
chunk, carving the packed blob differently on the two sides."""
|
||||||
"""
|
|
||||||
rec, fake = self._recorder_and_fake(shape=(4,))
|
rec, fake = self._recorder_and_fake(shape=(4,))
|
||||||
layer = _FakeLayer("reinterpreted")
|
layer = _FakeLayer("reinterpreted")
|
||||||
rec.current = (layer, "weight")
|
rec.current = (layer, "weight")
|
||||||
@@ -384,8 +374,7 @@ class TestBakeRecording:
|
|||||||
the producer packs what the chain yields, and nothing downstream can
|
the producer packs what the chain yields, and nothing downstream can
|
||||||
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
|
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
|
||||||
elements, so it reshapes cleanly over bytes laid out at other offsets.
|
elements, so it reshapes cleanly over bytes laid out at other offsets.
|
||||||
So the bake refuses rather than recording a slice it cannot carve.
|
So the bake refuses rather than recording a slice it cannot carve."""
|
||||||
"""
|
|
||||||
rec, fake = self._recorder_and_fake(shape=(1, 6))
|
rec, fake = self._recorder_and_fake(shape=(1, 6))
|
||||||
rec.current = (_FakeLayer("broadcast"), "weight")
|
rec.current = (_FakeLayer("broadcast"), "weight")
|
||||||
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
|
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
|
||||||
@@ -401,8 +390,7 @@ class TestBakeRecording:
|
|||||||
class TestConsumerIdentity:
|
class TestConsumerIdentity:
|
||||||
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
|
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
|
||||||
worker's producer block and keys the producer's per-consumer serve ring, so
|
worker's producer block and keys the producer's per-consumer serve ring, so
|
||||||
a collision silently serves two workers out of one ring.
|
a collision silently serves two workers out of one ring."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _engine(self, *, dp_index, rank, world_size):
|
def _engine(self, *, dp_index, rank, world_size):
|
||||||
eng = object.__new__(ShardedRDTWeightTransferEngine)
|
eng = object.__new__(ShardedRDTWeightTransferEngine)
|
||||||
@@ -439,8 +427,7 @@ class TestConsumerIdentity:
|
|||||||
def test_independent_engines_offset_into_distinct_ranges(self):
|
def test_independent_engines_offset_into_distinct_ranges(self):
|
||||||
"""A fleet of separate engines restarts _global_worker_index at 0 in
|
"""A fleet of separate engines restarts _global_worker_index at 0 in
|
||||||
each, so without the replica offset every engine would claim 0..w-1.
|
each, so without the replica offset every engine would claim 0..w-1.
|
||||||
The driver sets replica_rank on the payload; these are the ids it buys.
|
The driver sets replica_rank on the payload; these are the ids it buys."""
|
||||||
"""
|
|
||||||
ids = [
|
ids = [
|
||||||
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
|
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
|
||||||
for r in (0, 1)
|
for r in (0, 1)
|
||||||
@@ -466,8 +453,7 @@ def _router(
|
|||||||
group_lens=None,
|
group_lens=None,
|
||||||
):
|
):
|
||||||
"""A router over ``n`` single-name groups unless told otherwise, so a test
|
"""A router over ``n`` single-name groups unless told otherwise, so a test
|
||||||
can state expectations in group indices.
|
can state expectations in group indices."""
|
||||||
"""
|
|
||||||
if names is None:
|
if names is None:
|
||||||
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
|
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
|
||||||
names = [f"g{i}" for i in range(n)]
|
names = [f"g{i}" for i in range(n)]
|
||||||
@@ -496,8 +482,7 @@ class TestRdtRouter:
|
|||||||
def test_every_producer_nic_carries_traffic(self):
|
def test_every_producer_nic_carries_traffic(self):
|
||||||
"""16 producers / 8 consumers: the block rule spreads each consumer's
|
"""16 producers / 8 consumers: the block rule spreads each consumer's
|
||||||
pulls over its block, alternating by group, so no producer sits idle.
|
pulls over its block, alternating by group, so no producer sits idle.
|
||||||
Plain `consumer_id % len(owners)` would idle half of them.
|
Plain `consumer_id % len(owners)` would idle half of them."""
|
||||||
"""
|
|
||||||
r = _router(16, 8, group_lens=[1] * 95)
|
r = _router(16, 8, group_lens=[1] * 95)
|
||||||
names = [f"g{i}" for i in range(95)]
|
names = [f"g{i}" for i in range(95)]
|
||||||
for c in range(8):
|
for c in range(8):
|
||||||
@@ -515,8 +500,7 @@ class TestRdtRouter:
|
|||||||
|
|
||||||
def test_pipeline_stages_route_to_the_owning_stage(self):
|
def test_pipeline_stages_route_to_the_owning_stage(self):
|
||||||
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
|
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
|
||||||
sets, one per stage; each consumer reaches both.
|
sets, one per stage; each consumer reaches both."""
|
||||||
"""
|
|
||||||
r = _router(
|
r = _router(
|
||||||
16,
|
16,
|
||||||
8,
|
8,
|
||||||
@@ -532,8 +516,7 @@ class TestRdtRouter:
|
|||||||
|
|
||||||
def test_expert_names_route_to_the_holding_ranks(self):
|
def test_expert_names_route_to_the_holding_ranks(self):
|
||||||
"""A name held by only some ranks routes to those; a replicated name
|
"""A name held by only some ranks routes to those; a replicated name
|
||||||
keeps the full set.
|
keeps the full set."""
|
||||||
"""
|
|
||||||
r = RdtRouter(
|
r = RdtRouter(
|
||||||
8,
|
8,
|
||||||
4,
|
4,
|
||||||
@@ -552,8 +535,7 @@ class TestRdtRouter:
|
|||||||
|
|
||||||
def test_consumers_spread_over_an_owner_set(self):
|
def test_consumers_spread_over_an_owner_set(self):
|
||||||
"""Several ranks hold a name (its TP peers); the block rule must spread
|
"""Several ranks hold a name (its TP peers); the block rule must spread
|
||||||
consumers across them, not funnel through one.
|
consumers across them, not funnel through one."""
|
||||||
"""
|
|
||||||
r = _router(
|
r = _router(
|
||||||
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
|
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
|
||||||
)
|
)
|
||||||
@@ -562,8 +544,7 @@ class TestRdtRouter:
|
|||||||
|
|
||||||
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
|
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
|
||||||
"""The free barrier is per group, so its fan-out is the UNION over the
|
"""The free barrier is per group, so its fan-out is the UNION over the
|
||||||
group's names: a rank holding none of them is not signalled.
|
group's names: a rank holding none of them is not signalled."""
|
||||||
"""
|
|
||||||
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
|
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
|
||||||
assert r.group_owners(0) == [0, 1, 2, 3]
|
assert r.group_owners(0) == [0, 1, 2, 3]
|
||||||
assert r.group_owners(1) == [0, 1]
|
assert r.group_owners(1) == [0, 1]
|
||||||
@@ -633,8 +614,7 @@ class TestReplicaOverlay:
|
|||||||
|
|
||||||
def test_adding_a_deployment_does_not_move_the_first_one(self):
|
def test_adding_a_deployment_does_not_move_the_first_one(self):
|
||||||
"""The strongest form: an existing consumer's routes are byte-identical
|
"""The strongest form: an existing consumer's routes are byte-identical
|
||||||
before and after another deployment is provisioned.
|
before and after another deployment is provisioned."""
|
||||||
"""
|
|
||||||
one, names = self._fleet(16, 8, 1)
|
one, names = self._fleet(16, 8, 1)
|
||||||
two, _ = self._fleet(16, 8, 2)
|
two, _ = self._fleet(16, 8, 2)
|
||||||
for consumer_id in range(8):
|
for consumer_id in range(8):
|
||||||
@@ -644,8 +624,7 @@ class TestReplicaOverlay:
|
|||||||
|
|
||||||
def test_one_deployment_is_the_historical_carve(self):
|
def test_one_deployment_is_the_historical_carve(self):
|
||||||
"""Default (no width) and width == the whole fleet must agree with the
|
"""Default (no width) and width == the whole fleet must agree with the
|
||||||
plain block rule, so a single-deployment fleet is untouched.
|
plain block rule, so a single-deployment fleet is untouched."""
|
||||||
"""
|
|
||||||
plain = _router(16, 8, group_lens=[1] * 6)
|
plain = _router(16, 8, group_lens=[1] * 6)
|
||||||
explicit, names = self._fleet(16, 8, 1)
|
explicit, names = self._fleet(16, 8, 1)
|
||||||
for consumer_id in range(8):
|
for consumer_id in range(8):
|
||||||
@@ -659,8 +638,7 @@ class TestReplicaOverlay:
|
|||||||
def test_the_overlay_holds_for_partially_owned_names(self):
|
def test_the_overlay_holds_for_partially_owned_names(self):
|
||||||
"""Pipeline stages and experts too: two owner sets, one narrow. The
|
"""Pipeline stages and experts too: two owner sets, one narrow. The
|
||||||
narrow one already forced agreement (one owner, no block to carve); the
|
narrow one already forced agreement (one owner, no block to carve); the
|
||||||
wide one is what the carve decides.
|
wide one is what the carve decides."""
|
||||||
"""
|
|
||||||
r, names = self._fleet(
|
r, names = self._fleet(
|
||||||
8,
|
8,
|
||||||
4,
|
4,
|
||||||
@@ -676,16 +654,14 @@ class TestReplicaOverlay:
|
|||||||
def test_a_width_that_does_not_divide_the_fleet_raises(self):
|
def test_a_width_that_does_not_divide_the_fleet_raises(self):
|
||||||
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
|
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
|
||||||
is not, the overlay would map two workers of one deployment onto the same
|
is not, the overlay would map two workers of one deployment onto the same
|
||||||
block index, so refuse rather than serve the wrong bytes.
|
block index, so refuse rather than serve the wrong bytes."""
|
||||||
"""
|
|
||||||
with pytest.raises(ValueError, match="does not divide"):
|
with pytest.raises(ValueError, match="does not divide"):
|
||||||
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
|
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
|
||||||
|
|
||||||
def test_pulls_still_carry_the_global_consumer_id(self):
|
def test_pulls_still_carry_the_global_consumer_id(self):
|
||||||
"""The carve uses the index WITHIN a deployment, but the wire must keep
|
"""The carve uses the index WITHIN a deployment, but the wire must keep
|
||||||
the fleet-global id: it is what lets the producer tell the sharers of a
|
the fleet-global id: it is what lets the producer tell the sharers of a
|
||||||
slot apart, and count their arrivals separately.
|
slot apart, and count their arrivals separately."""
|
||||||
"""
|
|
||||||
|
|
||||||
class _Method:
|
class _Method:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -728,8 +704,7 @@ def _producer_pack_offsets(slices):
|
|||||||
class TestPackedLayout:
|
class TestPackedLayout:
|
||||||
def test_consumer_layout_matches_the_producer_rule(self):
|
def test_consumer_layout_matches_the_producer_rule(self):
|
||||||
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
|
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
|
||||||
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.
|
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment."""
|
||||||
"""
|
|
||||||
specs = [
|
specs = [
|
||||||
("w.fp8", "float8_e4m3fn", [17]),
|
("w.fp8", "float8_e4m3fn", [17]),
|
||||||
("w.scale", "float32", [3]),
|
("w.scale", "float32", [3]),
|
||||||
@@ -804,8 +779,7 @@ class TestPackedLayout:
|
|||||||
class TestChunkModuleScatters:
|
class TestChunkModuleScatters:
|
||||||
"""The chunk cut: one chunk per distinct owner class present in the copies,
|
"""The chunk cut: one chunk per distinct owner class present in the copies,
|
||||||
ascending by class index. Derived purely from the bake plus the ownership
|
ascending by class index. Derived purely from the bake plus the ownership
|
||||||
table, so vLLM's expert placement needs no cases.
|
table, so vLLM's expert placement needs no cases."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_unstamped_copies_form_one_chunk(self):
|
def test_unstamped_copies_form_one_chunk(self):
|
||||||
layer = _FakeLayer("l")
|
layer = _FakeLayer("l")
|
||||||
@@ -819,8 +793,7 @@ class TestChunkModuleScatters:
|
|||||||
"""Classes are numbered by first appearance in metadata order, so the
|
"""Classes are numbered by first appearance in metadata order, so the
|
||||||
replicated class (0) leads and the rest follow — the ordering the old
|
replicated class (0) leads and the rest follow — the ordering the old
|
||||||
"-1 first, then ascending stamp" rule produced, now falling out of the
|
"-1 first, then ascending stamp" rule produced, now falling out of the
|
||||||
table instead of a special case.
|
table instead of a special case."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("moe")
|
layer = _FakeLayer("moe")
|
||||||
names = ["norm", "e5", "e0", "e2"]
|
names = ["norm", "e5", "e0", "e2"]
|
||||||
copies = [_copy(n, shape=(4,)) for n in names]
|
copies = [_copy(n, shape=(4,)) for n in names]
|
||||||
@@ -840,8 +813,7 @@ class TestChunkModuleScatters:
|
|||||||
|
|
||||||
def test_copy_order_within_a_chunk_is_bake_order(self):
|
def test_copy_order_within_a_chunk_is_bake_order(self):
|
||||||
"""Scatter order within a chunk must follow the bake, not the stamp
|
"""Scatter order within a chunk must follow the bake, not the stamp
|
||||||
sort: the packed layout and the producer's replay agree on keys order.
|
sort: the packed layout and the producer's replay agree on keys order."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("moe")
|
layer = _FakeLayer("moe")
|
||||||
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
|
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
|
||||||
copies = [_copy(n, shape=(4,)) for n in names]
|
copies = [_copy(n, shape=(4,)) for n in names]
|
||||||
@@ -855,8 +827,7 @@ class TestChunkModuleScatters:
|
|||||||
|
|
||||||
def test_stamps_interleave_across_modules_without_reordering_within(self):
|
def test_stamps_interleave_across_modules_without_reordering_within(self):
|
||||||
"""Two modules' copies bucket by stamp; within each bucket the order is
|
"""Two modules' copies bucket by stamp; within each bucket the order is
|
||||||
module-then-bake order.
|
module-then-bake order."""
|
||||||
"""
|
|
||||||
a, b = _FakeLayer("a"), _FakeLayer("b")
|
a, b = _FakeLayer("a"), _FakeLayer("b")
|
||||||
mods = [
|
mods = [
|
||||||
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
|
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
|
||||||
@@ -876,8 +847,7 @@ class TestChunkModuleScatters:
|
|||||||
def test_scatters_carry_their_own_dtype(self):
|
def test_scatters_carry_their_own_dtype(self):
|
||||||
"""Dtype rides the record from the bake, where it is the fake's dtype
|
"""Dtype rides the record from the bake, where it is the fake's dtype
|
||||||
AFTER its op chain — not a plan-time lookup of the source name, which
|
AFTER its op chain — not a plan-time lookup of the source name, which
|
||||||
would be wrong for any chain that reinterprets dtype.
|
would be wrong for any chain that reinterprets dtype."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("l")
|
layer = _FakeLayer("l")
|
||||||
eng = _planner({}, name_meta={"w": ("float32", [10])})
|
eng = _planner({}, name_meta={"w": ("float32", [10])})
|
||||||
((_er, scatters),) = eng._chunk_module_scatters(
|
((_er, scatters),) = eng._chunk_module_scatters(
|
||||||
@@ -916,8 +886,7 @@ class TestBuildCallPlan:
|
|||||||
|
|
||||||
def test_materialize_fires_on_a_modules_first_chunk_only(self):
|
def test_materialize_fires_on_a_modules_first_chunk_only(self):
|
||||||
"""Empty HF params are allocated once per module, by construction —
|
"""Empty HF params are allocated once per module, by construction —
|
||||||
including a FusedMoE-like module whose copies span owner-class chunks.
|
including a FusedMoE-like module whose copies span owner-class chunks."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("spanning")
|
layer = _FakeLayer("spanning")
|
||||||
names = [f"w{i}" for i in range(4)]
|
names = [f"w{i}" for i in range(4)]
|
||||||
copies = [_copy(n, shape=(4,)) for n in names]
|
copies = [_copy(n, shape=(4,)) for n in names]
|
||||||
@@ -952,8 +921,7 @@ class TestBuildCallPlan:
|
|||||||
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
|
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
|
||||||
"""The load-bearing case: with the group cut across owner-class chunks the
|
"""The load-bearing case: with the group cut across owner-class chunks the
|
||||||
signal must hang off the LAST one. Signaling earlier lets the producers
|
signal must hang off the LAST one. Signaling earlier lets the producers
|
||||||
drop the gather buffers while a later chunk's RDMA is still reading.
|
drop the gather buffers while a later chunk's RDMA is still reading."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("l")
|
layer = _FakeLayer("l")
|
||||||
names = [f"w{i}" for i in range(4)]
|
names = [f"w{i}" for i in range(4)]
|
||||||
copies = [_copy(n, shape=(4,)) for n in names]
|
copies = [_copy(n, shape=(4,)) for n in names]
|
||||||
@@ -969,8 +937,7 @@ class TestBuildCallPlan:
|
|||||||
|
|
||||||
def test_free_signal_timing_across_two_split_groups(self):
|
def test_free_signal_timing_across_two_split_groups(self):
|
||||||
"""Two groups x 2 owner-class chunks each: signals land on each group's
|
"""Two groups x 2 owner-class chunks each: signals land on each group's
|
||||||
last chunk only.
|
last chunk only."""
|
||||||
"""
|
|
||||||
names_a = [f"model.layers.0.w{i}" for i in range(4)]
|
names_a = [f"model.layers.0.w{i}" for i in range(4)]
|
||||||
names_b = [f"model.layers.1.w{i}" for i in range(4)]
|
names_b = [f"model.layers.1.w{i}" for i in range(4)]
|
||||||
baked, name_meta, stamps = {}, {}, {}
|
baked, name_meta, stamps = {}, {}, {}
|
||||||
@@ -988,8 +955,7 @@ class TestBuildCallPlan:
|
|||||||
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
|
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
|
||||||
"""The owners still published it, so this consumer must signal it —
|
"""The owners still published it, so this consumer must signal it —
|
||||||
immediately, before the pipeline (signal-before-publish is tolerated),
|
immediately, before the pipeline (signal-before-publish is tolerated),
|
||||||
never hung off another group's chunk.
|
never hung off another group's chunk."""
|
||||||
"""
|
|
||||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||||
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
|
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
|
||||||
eng = _planner(
|
eng = _planner(
|
||||||
@@ -1010,8 +976,7 @@ class TestBuildCallPlan:
|
|||||||
|
|
||||||
def test_modules_are_deduped_within_a_group(self):
|
def test_modules_are_deduped_within_a_group(self):
|
||||||
"""Several names of one fused module map to the same scatter list; the
|
"""Several names of one fused module map to the same scatter list; the
|
||||||
plan must not chunk it twice.
|
plan must not chunk it twice."""
|
||||||
"""
|
|
||||||
layer = _FakeLayer("fused")
|
layer = _FakeLayer("fused")
|
||||||
names = ["qkv.q", "qkv.k", "qkv.v"]
|
names = ["qkv.q", "qkv.k", "qkv.v"]
|
||||||
copies = [_copy(n, shape=(4,)) for n in names]
|
copies = [_copy(n, shape=(4,)) for n in names]
|
||||||
@@ -1027,8 +992,7 @@ class TestBuildCallPlan:
|
|||||||
|
|
||||||
def test_plan_is_a_pure_function_of_its_inputs(self):
|
def test_plan_is_a_pure_function_of_its_inputs(self):
|
||||||
"""Called twice with the same arguments it must produce the same plan —
|
"""Called twice with the same arguments it must produce the same plan —
|
||||||
that is what lets the engine build it once and reuse it every sync.
|
that is what lets the engine build it once and reuse it every sync."""
|
||||||
"""
|
|
||||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||||
eng = _planner(baked, name_meta=name_meta)
|
eng = _planner(baked, name_meta=name_meta)
|
||||||
a = eng._build_call_plan(names, group_lens)
|
a = eng._build_call_plan(names, group_lens)
|
||||||
@@ -1048,8 +1012,7 @@ class TestBuildCallPlan:
|
|||||||
class TestUnbakedNamesGuard:
|
class TestUnbakedNamesGuard:
|
||||||
"""A live name with no baked plan has no way to load — there is no fallback,
|
"""A live name with no baked plan has no way to load — there is no fallback,
|
||||||
and its pull would target groups the pipeline already freed — so the plan
|
and its pull would target groups the pipeline already freed — so the plan
|
||||||
build fails at init, naming the names.
|
build fails at init, naming the names."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_a_live_unbaked_name_fails_the_plan_build(self):
|
def test_a_live_unbaked_name_fails_the_plan_build(self):
|
||||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||||
@@ -1060,8 +1023,7 @@ class TestUnbakedNamesGuard:
|
|||||||
|
|
||||||
def test_never_copied_names_are_dropped_entirely(self):
|
def test_never_copied_names_are_dropped_entirely(self):
|
||||||
"""Experts owned by another EP rank no-op in their loader: not an error,
|
"""Experts owned by another EP rank no-op in their loader: not an error,
|
||||||
and not a chunk — their group is pre-freed if nothing else fills it.
|
and not a chunk — their group is pre-freed if nothing else fills it."""
|
||||||
"""
|
|
||||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||||
del baked["norm.weight"]
|
del baked["norm.weight"]
|
||||||
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
|
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
|
||||||
@@ -1073,15 +1035,13 @@ class TestUnbakedNamesGuard:
|
|||||||
class TestCallPlanRouting:
|
class TestCallPlanRouting:
|
||||||
"""Under partial ownership each chunk must be pulled from a producer that
|
"""Under partial ownership each chunk must be pulled from a producer that
|
||||||
actually holds every name in it. Consumers bind EVERY producer (signals fan
|
actually holds every name in it. Consumers bind EVERY producer (signals fan
|
||||||
out to all owners), so local index == trainer rank.
|
out to all owners), so local index == trainer rank."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _routed_planner(
|
def _routed_planner(
|
||||||
self, held_by_group, num_consumers, consumer_id, *, held_by=None
|
self, held_by_group, num_consumers, consumer_id, *, held_by=None
|
||||||
):
|
):
|
||||||
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
|
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
|
||||||
can still speak in groups; ``held_by`` overrides individual names.
|
can still speak in groups; ``held_by`` overrides individual names."""
|
||||||
"""
|
|
||||||
baked, name_meta, names, group_lens = _one_module_per_layer(
|
baked, name_meta, names, group_lens = _one_module_per_layer(
|
||||||
len(held_by_group) - 2
|
len(held_by_group) - 2
|
||||||
)
|
)
|
||||||
@@ -1108,16 +1068,14 @@ class TestCallPlanRouting:
|
|||||||
|
|
||||||
def test_two_stage_ownership_splits_chunks_between_owners(self):
|
def test_two_stage_ownership_splits_chunks_between_owners(self):
|
||||||
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
|
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
|
||||||
second. Every chunk goes to a holder of its names.
|
second. Every chunk goes to a holder of its names."""
|
||||||
"""
|
|
||||||
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
|
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
|
||||||
plan = eng._build_call_plan(names, group_lens)
|
plan = eng._build_call_plan(names, group_lens)
|
||||||
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
|
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
|
||||||
|
|
||||||
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
|
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
|
||||||
"""Every chunk is served by exactly ONE producer: splitting a pull only
|
"""Every chunk is served by exactly ONE producer: splitting a pull only
|
||||||
multiplies produce calls, since the consumer's NIC bounds it.
|
multiplies produce calls, since the consumer's NIC bounds it."""
|
||||||
"""
|
|
||||||
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
|
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
|
||||||
plan = eng._build_call_plan(names, group_lens)
|
plan = eng._build_call_plan(names, group_lens)
|
||||||
assert all(0 <= c.owner < 2 for c in plan.chunks)
|
assert all(0 <= c.owner < 2 for c in plan.chunks)
|
||||||
@@ -1125,8 +1083,7 @@ class TestCallPlanRouting:
|
|||||||
def test_an_expert_name_routes_to_the_rank_holding_it(self):
|
def test_an_expert_name_routes_to_the_rank_holding_it(self):
|
||||||
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
|
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
|
||||||
name inside a stage-1 group is held by rank 3 alone. The name's own owner
|
name inside a stage-1 group is held by rank 3 alone. The name's own owner
|
||||||
set is what routes it.
|
set is what routes it."""
|
||||||
"""
|
|
||||||
eng, names, group_lens = self._routed_planner(
|
eng, names, group_lens = self._routed_planner(
|
||||||
[[0, 1], [0, 1], [2, 3], [2, 3]],
|
[[0, 1], [0, 1], [2, 3], [2, 3]],
|
||||||
1,
|
1,
|
||||||
@@ -1141,8 +1098,7 @@ class TestCallPlanRouting:
|
|||||||
def test_an_unheld_name_is_rejected_at_init(self):
|
def test_an_unheld_name_is_rejected_at_init(self):
|
||||||
"""A name no rank holds can never be served. The trainer raises when it
|
"""A name no rank holds can never be served. The trainer raises when it
|
||||||
transposes the holdings; the router's validate() is the consumer-side
|
transposes the holdings; the router's validate() is the consumer-side
|
||||||
backstop, and either way it fails at init rather than at first pull.
|
backstop, and either way it fails at init rather than at first pull."""
|
||||||
"""
|
|
||||||
with pytest.raises(ValueError, match="empty"):
|
with pytest.raises(ValueError, match="empty"):
|
||||||
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
|
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
|
||||||
|
|
||||||
@@ -1151,16 +1107,14 @@ class TestSignalCompleteness:
|
|||||||
"""Every gather group is signaled exactly once — after its last chunk when
|
"""Every gather group is signaled exactly once — after its last chunk when
|
||||||
the worker pulls from it, at sync start otherwise — for ANY expert
|
the worker pulls from it, at sync start otherwise — for ANY expert
|
||||||
placement. A group signaled twice over-credits the barrier; a group never
|
placement. A group signaled twice over-credits the barrier; a group never
|
||||||
signaled parks a producer credit and hangs end_sync.
|
signaled parks a producer credit and hangs end_sync."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
|
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
|
||||||
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
|
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
|
||||||
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
|
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
|
||||||
The worker's bake covers only ``worker_experts`` (its placement);
|
The worker's bake covers only ``worker_experts`` (its placement);
|
||||||
foreign expert names never copied => dropped entirely, as in the real
|
foreign expert names never copied => dropped entirely, as in the real
|
||||||
bake.
|
bake."""
|
||||||
"""
|
|
||||||
n_local = n_experts // ep_size
|
n_local = n_experts // ep_size
|
||||||
names = ["embed.weight"]
|
names = ["embed.weight"]
|
||||||
group_lens = [1]
|
group_lens = [1]
|
||||||
@@ -1220,8 +1174,7 @@ class TestSignalCompleteness:
|
|||||||
|
|
||||||
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
|
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
|
||||||
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
||||||
hits all 4. More chunks per group, identical signal set.
|
hits all 4. More chunks per group, identical signal set."""
|
||||||
"""
|
|
||||||
plans = {}
|
plans = {}
|
||||||
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
|
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
|
||||||
eng, names, group_lens = self._moe_planner(worker_experts=experts)
|
eng, names, group_lens = self._moe_planner(worker_experts=experts)
|
||||||
@@ -1235,8 +1188,7 @@ class TestSignalCompleteness:
|
|||||||
|
|
||||||
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
|
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
|
||||||
"""The FusedMoE shape: one module's copies span every owner-class chunk of
|
"""The FusedMoE shape: one module's copies span every owner-class chunk of
|
||||||
its group; materialize on its first, quant on its last.
|
its group; materialize on its first, quant on its last."""
|
||||||
"""
|
|
||||||
eng, names, group_lens = self._moe_planner(
|
eng, names, group_lens = self._moe_planner(
|
||||||
worker_experts=[0, 2, 4, 6], n_layers=1
|
worker_experts=[0, 2, 4, 6], n_layers=1
|
||||||
)
|
)
|
||||||
@@ -1294,8 +1246,7 @@ class TestLayerwiseGroups:
|
|||||||
def test_flattening_a_partition_returns_the_input_order(self):
|
def test_flattening_a_partition_returns_the_input_order(self):
|
||||||
"""Un-indexed names group by POSITION, not by name class — so one that
|
"""Un-indexed names group by POSITION, not by name class — so one that
|
||||||
looks like a pre-block name still lands after the layers it follows.
|
looks like a pre-block name still lands after the layers it follows.
|
||||||
Group index therefore means the same thing on every rank.
|
Group index therefore means the same thing on every rank."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"embed.weight",
|
"embed.weight",
|
||||||
"model.layers.0.a",
|
"model.layers.0.a",
|
||||||
@@ -1313,8 +1264,7 @@ class TestLayerwiseGroups:
|
|||||||
|
|
||||||
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
|
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
|
||||||
"""Two stacks stay in the order they appear — the vision tower before the
|
"""Two stacks stay in the order they appear — the vision tower before the
|
||||||
text stack — while each sorts by index internally.
|
text stack — while each sorts by index internally."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"visual.blocks.1.w",
|
"visual.blocks.1.w",
|
||||||
"visual.blocks.0.w",
|
"visual.blocks.0.w",
|
||||||
@@ -1331,8 +1281,7 @@ class TestLayerwiseGroups:
|
|||||||
def test_the_post_block_lands_last_however_early_it_arrives(self):
|
def test_the_post_block_lands_last_however_early_it_arrives(self):
|
||||||
"""Megatron-Bridge streams the last pipeline stage's output block before
|
"""Megatron-Bridge streams the last pipeline stage's output block before
|
||||||
its layers. Sweeping un-indexed names after the first layer into a
|
its layers. Sweeping un-indexed names after the first layer into a
|
||||||
trailing group is what lets the gather loop walk groups ascending.
|
trailing group is what lets the gather loop walk groups ascending."""
|
||||||
"""
|
|
||||||
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
|
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
|
||||||
assert layerwise_groups(names) == [
|
assert layerwise_groups(names) == [
|
||||||
["model.layers.0.w"],
|
["model.layers.0.w"],
|
||||||
@@ -1342,8 +1291,7 @@ class TestLayerwiseGroups:
|
|||||||
|
|
||||||
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
|
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
|
||||||
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
|
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
|
||||||
the key keeps that from shattering one layer into many groups.
|
the key keeps that from shattering one layer into many groups."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"model.layers.0.a",
|
"model.layers.0.a",
|
||||||
"model.layers.1.a",
|
"model.layers.1.a",
|
||||||
@@ -1385,15 +1333,13 @@ class TestLayerwiseGroups:
|
|||||||
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
|
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
|
||||||
"""One decoder layer per group under any prefix, with no per-architecture
|
"""One decoder layer per group under any prefix, with no per-architecture
|
||||||
table. A literal `model.layers.` match silently yields one whole-model
|
table. A literal `model.layers.` match silently yields one whole-model
|
||||||
group for most of these.
|
group for most of these."""
|
||||||
"""
|
|
||||||
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
|
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
|
||||||
assert layerwise_groups(names) == [[n] for n in names]
|
assert layerwise_groups(names) == [[n] for n in names]
|
||||||
|
|
||||||
def test_a_moe_layer_stays_whole(self):
|
def test_a_moe_layer_stays_whole(self):
|
||||||
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
|
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
|
||||||
on the expert would cut FusedMoE — one leaf module — across groups.
|
on the expert would cut FusedMoE — one leaf module — across groups."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"model.layers.0.mlp.experts.0.w1.weight",
|
"model.layers.0.mlp.experts.0.w1.weight",
|
||||||
"model.layers.0.mlp.experts.1.w1.weight",
|
"model.layers.0.mlp.experts.1.w1.weight",
|
||||||
@@ -1410,8 +1356,7 @@ class TestLayerwiseGroups:
|
|||||||
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
|
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
|
||||||
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
|
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
|
||||||
these in ONE group — the whole model — because the text stack ships as
|
these in ONE group — the whole model — because the text stack ships as
|
||||||
`model.language_model.layers.`.
|
`model.language_model.layers.`."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"visual.patch_embed.proj.weight",
|
"visual.patch_embed.proj.weight",
|
||||||
"visual.blocks.0.attn.qkv.weight",
|
"visual.blocks.0.attn.qkv.weight",
|
||||||
@@ -1430,8 +1375,7 @@ class TestLayerwiseGroups:
|
|||||||
|
|
||||||
def test_two_stacks_sharing_an_index_do_not_merge(self):
|
def test_two_stacks_sharing_an_index_do_not_merge(self):
|
||||||
"""The key carries the prefix, so vision block 0 and text layer 0 are
|
"""The key carries the prefix, so vision block 0 and text layer 0 are
|
||||||
different groups even though both are index 0.
|
different groups even though both are index 0."""
|
||||||
"""
|
|
||||||
names = [
|
names = [
|
||||||
"visual.blocks.0.attn.qkv.weight",
|
"visual.blocks.0.attn.qkv.weight",
|
||||||
"model.language_model.layers.0.self_attn.q_proj.weight",
|
"model.language_model.layers.0.self_attn.q_proj.weight",
|
||||||
@@ -1468,23 +1412,20 @@ class TestOpAllowlistAgreement:
|
|||||||
|
|
||||||
def test_every_bake_recordable_op_is_replayable(self):
|
def test_every_bake_recordable_op_is_replayable(self):
|
||||||
"""Anything the bake can record must be serveable, or the failure lands a
|
"""Anything the bake can record must be serveable, or the failure lands a
|
||||||
whole sync later than the mistake.
|
whole sync later than the mistake."""
|
||||||
"""
|
|
||||||
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
|
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
|
||||||
|
|
||||||
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
|
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
|
||||||
"""The allowlist is also a guard against a misbehaving or spoofed
|
"""The allowlist is also a guard against a misbehaving or spoofed
|
||||||
consumer invoking arbitrary methods on trainer tensors, so it must not be
|
consumer invoking arbitrary methods on trainer tensors, so it must not be
|
||||||
wider than the ops the bake can actually produce. ``to`` in particular
|
wider than the ops the bake can actually produce. ``to`` in particular
|
||||||
would let a replay change dtype or device.
|
would let a replay change dtype or device."""
|
||||||
"""
|
|
||||||
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
|
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
|
||||||
assert "to" not in ALLOWED_OPS
|
assert "to" not in ALLOWED_OPS
|
||||||
|
|
||||||
def test_transpose_via_t_is_serveable(self):
|
def test_transpose_via_t_is_serveable(self):
|
||||||
"""The specific regression: a chain recorded from ``.t()`` must pass the
|
"""The specific regression: a chain recorded from ``.t()`` must pass the
|
||||||
producer's guard.
|
producer's guard."""
|
||||||
"""
|
|
||||||
fake = FakeRDTTensor(
|
fake = FakeRDTTensor(
|
||||||
name="w",
|
name="w",
|
||||||
shape=torch.Size((4, 6)),
|
shape=torch.Size((4, 6)),
|
||||||
@@ -1529,8 +1470,7 @@ BAKE_MODELS = [
|
|||||||
class TestRequiresTheRayExecutor:
|
class TestRequiresTheRayExecutor:
|
||||||
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
|
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
|
||||||
check is at construction because the alternative is an opaque failure during
|
check is at construction because the alternative is an opaque failure during
|
||||||
the first handshake, long after the misconfiguration.
|
the first handshake, long after the misconfiguration."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _construct(self, backend):
|
def _construct(self, backend):
|
||||||
cfg = SimpleNamespace(
|
cfg = SimpleNamespace(
|
||||||
@@ -1557,8 +1497,7 @@ class TestRequiresTheRayExecutor:
|
|||||||
|
|
||||||
def test_a_custom_executor_class_is_left_alone(self):
|
def test_a_custom_executor_class_is_left_alone(self):
|
||||||
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
|
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
|
||||||
must not be rejected for merely not being the string "ray".
|
must not be rejected for merely not being the string "ray"."""
|
||||||
"""
|
|
||||||
|
|
||||||
class _CustomExecutor:
|
class _CustomExecutor:
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -236,8 +236,7 @@ class TestPublishAndRebuild:
|
|||||||
def test_views_of_one_storage_do_not_overlap(self, server_factory):
|
def test_views_of_one_storage_do_not_overlap(self, server_factory):
|
||||||
"""One IPC export per storage, one as_strided view per name — the whole
|
"""One IPC export per storage, one as_strided view per name — the whole
|
||||||
point of the storage/view split. Overlapping views would serve the same
|
point of the storage/view split. Overlapping views would serve the same
|
||||||
bytes for two different weights.
|
bytes for two different weights."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
_publish(server, GI_A, GROUP_A, nbytes=64)
|
_publish(server, GI_A, GROUP_A, nbytes=64)
|
||||||
@@ -260,8 +259,7 @@ class TestPublishAndRebuild:
|
|||||||
def test_the_default_lookahead_is_one(self):
|
def test_the_default_lookahead_is_one(self):
|
||||||
"""1 is the sweet spot: group N+1 is gathered AND pullable while N is
|
"""1 is the sweet spot: group N+1 is gathered AND pullable while N is
|
||||||
being pulled, with resident memory at its floor of 2 groups — the bound
|
being pulled, with resident memory at its floor of 2 groups — the bound
|
||||||
the larger-model runs size against.
|
the larger-model runs size against."""
|
||||||
"""
|
|
||||||
assert DEFAULT_GATHER_LOOKAHEAD == 1
|
assert DEFAULT_GATHER_LOOKAHEAD == 1
|
||||||
|
|
||||||
|
|
||||||
@@ -276,8 +274,7 @@ class TestFreeBarrier:
|
|||||||
|
|
||||||
def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory):
|
def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory):
|
||||||
"""Every live consumer signals every owner; releasing on the first
|
"""Every live consumer signals every owner; releasing on the first
|
||||||
signal would drop storage other consumers are still reading.
|
signal would drop storage other consumers are still reading."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(3)
|
server.begin_sync(3)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -304,8 +301,7 @@ class TestFreeBarrier:
|
|||||||
):
|
):
|
||||||
"""A consumer with nothing to pull from a group signals it at sync
|
"""A consumer with nothing to pull from a group signals it at sync
|
||||||
start, which can precede the publish. The publish must then release the
|
start, which can precede the publish. The publish must then release the
|
||||||
group rather than wait for a signal that will never come again.
|
group rather than wait for a signal that will never come again."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
server.free_group(GI_A)
|
server.free_group(GI_A)
|
||||||
@@ -328,8 +324,7 @@ class TestFreeBarrier:
|
|||||||
the server says the group is gone — and it hears that ONLY through
|
the server says the group is gone — and it hears that ONLY through
|
||||||
``wait_freed`` / ``end_sync``, never a publish return (a freed notice
|
``wait_freed`` / ``end_sync``, never a publish return (a freed notice
|
||||||
riding an unharvested async publish while the engine blocks in
|
riding an unharvested async publish while the engine blocks in
|
||||||
``wait_freed`` would wedge the loop).
|
``wait_freed`` would wedge the loop)."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -358,8 +353,7 @@ class TestFreeBarrier:
|
|||||||
|
|
||||||
class _CountingSource:
|
class _CountingSource:
|
||||||
"""`iter_groups` yielding one CPU-tensor group per call, built lazily so a
|
"""`iter_groups` yielding one CPU-tensor group per call, built lazily so a
|
||||||
group's tensors exist only once the engine's credit gate lets it gather.
|
group's tensors exist only once the engine's credit gate lets it gather."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, groups):
|
def __init__(self, groups):
|
||||||
self._groups = groups
|
self._groups = groups
|
||||||
@@ -372,8 +366,7 @@ class _CountingSource:
|
|||||||
class _ResidencyDict(dict):
|
class _ResidencyDict(dict):
|
||||||
"""The engine's `_inflight`, instrumented: `max_len` is the high-water mark
|
"""The engine's `_inflight`, instrumented: `max_len` is the high-water mark
|
||||||
of groups whose CUDA-IPC export refs were alive at once — the trainer's
|
of groups whose CUDA-IPC export refs were alive at once — the trainer's
|
||||||
resident-memory bound in groups.
|
resident-memory bound in groups."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -387,8 +380,7 @@ class _ResidencyDict(dict):
|
|||||||
def _loop_engine(server, n_groups, *, lookahead):
|
def _loop_engine(server, n_groups, *, lookahead):
|
||||||
"""A real engine wired straight to a real (non-Ray) server, with only the
|
"""A real engine wired straight to a real (non-Ray) server, with only the
|
||||||
state `_run_gather_loop` reads. The `_rpc` seam dispatches inline;
|
state `_run_gather_loop` reads. The `_rpc` seam dispatches inline;
|
||||||
`_publish_async` sees no `.remote` and runs inline too.
|
`_publish_async` sees no `.remote` and runs inline too."""
|
||||||
"""
|
|
||||||
groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)]
|
groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)]
|
||||||
e = ShardedRDTTrainerWeightTransferEngine.__new__(
|
e = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||||
ShardedRDTTrainerWeightTransferEngine
|
ShardedRDTTrainerWeightTransferEngine
|
||||||
@@ -411,16 +403,14 @@ class TestGatherCredit:
|
|||||||
serveable immediately, and the ENGINE's loop stops gathering while more than
|
serveable immediately, and the ENGINE's loop stops gathering while more than
|
||||||
`gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are
|
`gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are
|
||||||
resident — two at the default — while group N+1 is pulled the instant N's
|
resident — two at the default — while group N+1 is pulled the instant N's
|
||||||
pulls finish.
|
pulls finish."""
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def gather_engine(self, monkeypatch):
|
def gather_engine(self, monkeypatch):
|
||||||
"""Neutralize the two CUDA touches in the export path so the REAL
|
"""Neutralize the two CUDA touches in the export path so the REAL
|
||||||
gather loop runs on CPU: `.cuda()` becomes identity, and the storage
|
gather loop runs on CPU: `.cuda()` becomes identity, and the storage
|
||||||
export ships the `(nbytes, ..., device, ...)` tuple the fixture's
|
export ships the `(nbytes, ..., device, ...)` tuple the fixture's
|
||||||
stubbed `rebuild_cuda_tensor` expects.
|
stubbed `rebuild_cuda_tensor` expects."""
|
||||||
"""
|
|
||||||
monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self)
|
monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
trainer_mod,
|
trainer_mod,
|
||||||
@@ -432,8 +422,7 @@ class TestGatherCredit:
|
|||||||
def test_publish_never_blocks(self, server_factory):
|
def test_publish_never_blocks(self, server_factory):
|
||||||
"""Publishing must not park on a credit: the whole point is that a
|
"""Publishing must not park on a credit: the whole point is that a
|
||||||
gathered group becomes serveable immediately. Three publishes with no
|
gathered group becomes serveable immediately. Three publishes with no
|
||||||
frees, inline on this thread — a block here hangs the test.
|
frees, inline on this thread — a block here hangs the test."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1)
|
server = server_factory(gather_lookahead=1)
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
for gi in range(3):
|
for gi in range(3):
|
||||||
@@ -447,8 +436,7 @@ class TestGatherCredit:
|
|||||||
than 2 groups of gathered weights, no matter how the consumer paces —
|
than 2 groups of gathered weights, no matter how the consumer paces —
|
||||||
and the pipeline still overlaps, because the consumer here refuses to
|
and the pipeline still overlaps, because the consumer here refuses to
|
||||||
free group N until N+1 is already published (pullable). A gate an
|
free group N until N+1 is already published (pullable). A gate an
|
||||||
off-by-one too tight deadlocks this test; too loose fails max_len.
|
off-by-one too tight deadlocks this test; too loose fails max_len."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1)
|
server = server_factory(gather_lookahead=1)
|
||||||
n_groups = 8
|
n_groups = 8
|
||||||
engine = gather_engine(server, n_groups, lookahead=1)
|
engine = gather_engine(server, n_groups, lookahead=1)
|
||||||
@@ -490,8 +478,7 @@ class TestGatherCredit:
|
|||||||
):
|
):
|
||||||
"""0 is the serialized baseline: the loop waits for group N to be freed
|
"""0 is the serialized baseline: the loop waits for group N to be freed
|
||||||
before gathering N+1, so exactly ONE group is ever resident and nothing
|
before gathering N+1, so exactly ONE group is ever resident and nothing
|
||||||
overlaps. A floor of 1 on the bound would silently pipeline instead.
|
overlaps. A floor of 1 on the bound would silently pipeline instead."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=0)
|
server = server_factory(gather_lookahead=0)
|
||||||
engine = gather_engine(server, 4, lookahead=0)
|
engine = gather_engine(server, 4, lookahead=0)
|
||||||
|
|
||||||
@@ -517,8 +504,7 @@ class TestGatherCredit:
|
|||||||
self, server_factory, gather_engine
|
self, server_factory, gather_engine
|
||||||
):
|
):
|
||||||
"""Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
"""Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
||||||
consumer sees exactly 3 groups gathered before the loop parks.
|
consumer sees exactly 3 groups gathered before the loop parks."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=2)
|
server = server_factory(gather_lookahead=2)
|
||||||
engine = gather_engine(server, 8, lookahead=2)
|
engine = gather_engine(server, 8, lookahead=2)
|
||||||
|
|
||||||
@@ -554,8 +540,7 @@ class TestGatherCredit:
|
|||||||
def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory):
|
def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory):
|
||||||
"""Otherwise a trainer-side failure on another rank deadlocks this one
|
"""Otherwise a trainer-side failure on another rank deadlocks this one
|
||||||
inside its credit gate. It must RAISE, not return empty — an empty
|
inside its credit gate. It must RAISE, not return empty — an empty
|
||||||
return would spin the engine straight back into the wait.
|
return would spin the engine straight back into the wait."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -636,8 +621,7 @@ class TestBeginSync:
|
|||||||
|
|
||||||
def test_the_target_is_floored_at_one(self, server_factory):
|
def test_the_target_is_floored_at_one(self, server_factory):
|
||||||
"""A bare/zero call must not make every group free instantly (or divide
|
"""A bare/zero call must not make every group free instantly (or divide
|
||||||
the barrier by zero).
|
the barrier by zero)."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(0)
|
server.begin_sync(0)
|
||||||
assert server._live_count == 1
|
assert server._live_count == 1
|
||||||
@@ -645,8 +629,7 @@ class TestBeginSync:
|
|||||||
def test_a_degraded_sync_lowers_the_target(self, server_factory):
|
def test_a_degraded_sync_lowers_the_target(self, server_factory):
|
||||||
"""FT degraded sync: the live count is the WHOLE degraded-sync
|
"""FT degraded sync: the live count is the WHOLE degraded-sync
|
||||||
mechanism on this side — the group releases after the live consumers
|
mechanism on this side — the group releases after the live consumers
|
||||||
alone.
|
alone."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(2) # 4 provisioned, 2 alive
|
server.begin_sync(2) # 4 provisioned, 2 alive
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -670,8 +653,7 @@ class TestBeginSync:
|
|||||||
def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory):
|
def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory):
|
||||||
"""begin_sync resets the counts, so a signal that leaked past the sync
|
"""begin_sync resets the counts, so a signal that leaked past the sync
|
||||||
boundary would over-credit — the reason the consumer drains its fired
|
boundary would over-credit — the reason the consumer drains its fired
|
||||||
signals before finishing. Pin the reset side of that contract.
|
signals before finishing. Pin the reset side of that contract."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(2)
|
server.begin_sync(2)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -693,8 +675,7 @@ class TestBeginSync:
|
|||||||
self, server_factory
|
self, server_factory
|
||||||
):
|
):
|
||||||
"""The packed layout repeats every sync — that is what makes caching the
|
"""The packed layout repeats every sync — that is what makes caching the
|
||||||
destination views worth ~5ms per 384-spec group.
|
destination views worth ~5ms per 384-spec group."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server._pack_dsts[("sentinel",)] = (0, [])
|
server._pack_dsts[("sentinel",)] = (0, [])
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
@@ -705,8 +686,7 @@ class TestServedNamesGuard:
|
|||||||
"""A producer serves only the names it holds: its stage's groups, and
|
"""A producer serves only the names it holds: its stage's groups, and
|
||||||
within them its own EP coordinate's experts plus the replicated names.
|
within them its own EP coordinate's experts plus the replicated names.
|
||||||
Without this guard a misrouted pull would block forever in the cache wait
|
Without this guard a misrouted pull would block forever in the cache wait
|
||||||
for a name this rank never publishes.
|
for a name this rank never publishes."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory):
|
def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory):
|
||||||
server = server_factory(served_names=list(GROUP_A))
|
server = server_factory(served_names=list(GROUP_A))
|
||||||
@@ -722,8 +702,7 @@ class TestServedNamesGuard:
|
|||||||
|
|
||||||
def test_served_names_none_accepts_anything(self, server_factory):
|
def test_served_names_none_accepts_anything(self, server_factory):
|
||||||
"""Gather-to-all: every producer holds every group, so there is nothing
|
"""Gather-to-all: every producer holds every group, so there is nothing
|
||||||
to guard. The call proceeds to the cache wait (not exercised here).
|
to guard. The call proceeds to the cache wait (not exercised here)."""
|
||||||
"""
|
|
||||||
server = server_factory(served_names=None)
|
server = server_factory(served_names=None)
|
||||||
assert server._served_names is None
|
assert server._served_names is None
|
||||||
|
|
||||||
@@ -770,8 +749,7 @@ class TestStallWatchdog:
|
|||||||
|
|
||||||
def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory):
|
def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory):
|
||||||
"""The exact mid-sync death signature: the group is published, one of
|
"""The exact mid-sync death signature: the group is published, one of
|
||||||
the two live consumers that owed a signal is gone.
|
the two live consumers that owed a signal is gone."""
|
||||||
"""
|
|
||||||
server = server_factory(stall_timeout_s=0.3)
|
server = server_factory(stall_timeout_s=0.3)
|
||||||
server.begin_sync(2)
|
server.begin_sync(2)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -785,8 +763,7 @@ class TestStallWatchdog:
|
|||||||
"""The watchdog fires on the existing `set_gather_error` channel, so one
|
"""The watchdog fires on the existing `set_gather_error` channel, so one
|
||||||
stall unwinds the whole rank through one path rather than each waiter
|
stall unwinds the whole rank through one path rather than each waiter
|
||||||
timing out separately — here both a consumer parked in the serve cache
|
timing out separately — here both a consumer parked in the serve cache
|
||||||
wait and the engine parked in its credit gate.
|
wait and the engine parked in its credit gate."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -819,8 +796,7 @@ class TestStallWatchdog:
|
|||||||
"""A consumer that is slow but signaling must never trip it: the stamp
|
"""A consumer that is slow but signaling must never trip it: the stamp
|
||||||
is global to the producer, so a steady trickle of signals holds the
|
is global to the producer, so a steady trickle of signals holds the
|
||||||
whole rank open. Nine sequential publishes with a 0.3s timeout each take
|
whole rank open. Nine sequential publishes with a 0.3s timeout each take
|
||||||
longer in total than the timeout, and none may fire.
|
longer in total than the timeout, and none may fire."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
for gi in range(9):
|
for gi in range(9):
|
||||||
@@ -833,8 +809,7 @@ class TestStallWatchdog:
|
|||||||
|
|
||||||
def test_begin_sync_resets_the_progress_stamp(self, server_factory):
|
def test_begin_sync_resets_the_progress_stamp(self, server_factory):
|
||||||
"""Syncs are minutes apart. Without the reset the first publish of sync N+1
|
"""Syncs are minutes apart. Without the reset the first publish of sync N+1
|
||||||
would measure its stall from somewhere inside sync N and fire immediately.
|
would measure its stall from somewhere inside sync N and fire immediately."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -856,8 +831,7 @@ class TestConcurrentPublishAndFree:
|
|||||||
self, server_factory
|
self, server_factory
|
||||||
):
|
):
|
||||||
"""Every group published must end up freed and out of the cache, with
|
"""Every group published must end up freed and out of the cache, with
|
||||||
end_sync returning cleanly — the property whose violation is a hang.
|
end_sync returning cleanly — the property whose violation is a hang."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=2)
|
server = server_factory(gather_lookahead=2)
|
||||||
server.begin_sync(1)
|
server.begin_sync(1)
|
||||||
groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)]
|
groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)]
|
||||||
@@ -892,8 +866,7 @@ class TestConcurrentPublishAndFree:
|
|||||||
):
|
):
|
||||||
"""All live consumers signaling at once: the release must happen on the
|
"""All live consumers signaling at once: the release must happen on the
|
||||||
last signal and only once, or the engine drops IPC refs twice (or
|
last signal and only once, or the engine drops IPC refs twice (or
|
||||||
never).
|
never)."""
|
||||||
"""
|
|
||||||
server = server_factory()
|
server = server_factory()
|
||||||
server.begin_sync(8)
|
server.begin_sync(8)
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -920,8 +893,7 @@ class TestFakeServerAgreesWithTheRealOne:
|
|||||||
"""`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second,
|
"""`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second,
|
||||||
independent model of this protocol that the engine-side tests assert on. Pin
|
independent model of this protocol that the engine-side tests assert on. Pin
|
||||||
the two against each other so the fake cannot silently drift from the
|
the two against each other so the fake cannot silently drift from the
|
||||||
semantics the real server enforces.
|
semantics the real server enforces."""
|
||||||
"""
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _fake():
|
def _fake():
|
||||||
@@ -952,8 +924,7 @@ class TestExportRing:
|
|||||||
"""The packed export path. Its one hard invariant is that the trainer packs
|
"""The packed export path. Its one hard invariant is that the trainer packs
|
||||||
a group exactly the way ``publish_group`` rebuilds it — a layout
|
a group exactly the way ``publish_group`` rebuilds it — a layout
|
||||||
disagreement would serve the wrong bytes with nothing downstream to catch
|
disagreement would serve the wrong bytes with nothing downstream to catch
|
||||||
it.
|
it."""
|
||||||
"""
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _engine():
|
def _engine():
|
||||||
@@ -1045,8 +1016,7 @@ class TestExportRingSlotSafety:
|
|||||||
):
|
):
|
||||||
"""Free the later group of each pair first and hold the earlier one live
|
"""Free the later group of each pair first and hold the earlier one live
|
||||||
until the NEXT group is gathered -- the window a modular slot counter got
|
until the NEXT group is gathered -- the window a modular slot counter got
|
||||||
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's).
|
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's)."""
|
||||||
"""
|
|
||||||
server = server_factory(gather_lookahead=1)
|
server = server_factory(gather_lookahead=1)
|
||||||
n_groups = 8
|
n_groups = 8
|
||||||
engine = gather_engine(server, n_groups, lookahead=1)
|
engine = gather_engine(server, n_groups, lookahead=1)
|
||||||
@@ -1170,8 +1140,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
def test_one_deployment_never_shares(self, sharing_server):
|
def test_one_deployment_never_shares(self, sharing_server):
|
||||||
"""With the width equal to the live count every group is a singleton, so
|
"""With the width equal to the live count every group is a singleton, so
|
||||||
each call packs for itself out of its own ring.
|
each call packs for itself out of its own ring."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.begin_sync(2, [0, 1])
|
server.begin_sync(2, [0, 1])
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -1251,8 +1220,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
def test_a_seqless_call_is_refused(self, sharing_server):
|
def test_a_seqless_call_is_refused(self, sharing_server):
|
||||||
"""The slot comes from ``seq``, so a caller that sends none cannot be
|
"""The slot comes from ``seq``, so a caller that sends none cannot be
|
||||||
served safely.
|
served safely."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.begin_sync(1, [0])
|
server.begin_sync(1, [0])
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -1262,8 +1230,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
def test_a_failed_pack_reaches_every_sharer(self, sharing_server):
|
def test_a_failed_pack_reaches_every_sharer(self, sharing_server):
|
||||||
"""Only the packer touches the buffer, so its failure has to be
|
"""Only the packer touches the buffer, so its failure has to be
|
||||||
republished: a waiter must raise rather than read a half-written slot.
|
republished: a waiter must raise rather than read a half-written slot."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.begin_sync(4, [0, 1, 2, 3])
|
server.begin_sync(4, [0, 1, 2, 3])
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -1287,8 +1254,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server):
|
def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server):
|
||||||
"""A dead deployment must not be waited for: a survivor left alone in
|
"""A dead deployment must not be waited for: a survivor left alone in
|
||||||
its group packs for itself instead of blocking on the corpse.
|
its group packs for itself instead of blocking on the corpse."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0)
|
server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0)
|
||||||
server.begin_sync(3, [0, 1, 3]) # consumer 2 died
|
server.begin_sync(3, [0, 1, 3]) # consumer 2 died
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -1302,8 +1268,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server):
|
def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server):
|
||||||
"""A consumer that dies INSIDE the sync window has no detector, exactly
|
"""A consumer that dies INSIDE the sync window has no detector, exactly
|
||||||
as for the free barrier; the rendezvous waits on the same channel.
|
as for the free barrier; the rendezvous waits on the same channel."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3)
|
server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3)
|
||||||
server.begin_sync(4, [0, 1, 2, 3])
|
server.begin_sync(4, [0, 1, 2, 3])
|
||||||
_publish(server, GI_A, GROUP_A)
|
_publish(server, GI_A, GROUP_A)
|
||||||
@@ -1315,8 +1280,7 @@ class TestSharedServeSlots:
|
|||||||
|
|
||||||
class TestSharedReservation:
|
class TestSharedReservation:
|
||||||
"""``reserve_serve_buffer`` under sharing: one ring per GROUP, and the
|
"""``reserve_serve_buffer`` under sharing: one ring per GROUP, and the
|
||||||
init-time check that the group's members pull the same chunks.
|
init-time check that the group's members pull the same chunks."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_matching_plans_reserve_one_ring_between_them(self, sharing_server):
|
def test_matching_plans_reserve_one_ring_between_them(self, sharing_server):
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
@@ -1338,8 +1302,7 @@ class TestSharedReservation:
|
|||||||
def test_mismatched_plans_fail_at_reservation(self, sharing_server):
|
def test_mismatched_plans_fail_at_reservation(self, sharing_server):
|
||||||
"""Sharing rests on the sharers pulling the same chunks. Checked here,
|
"""Sharing rests on the sharers pulling the same chunks. Checked here,
|
||||||
where it names both consumers, rather than at serve time, where it is a
|
where it names both consumers, rather than at serve time, where it is a
|
||||||
rendezvous nobody completes.
|
rendezvous nobody completes."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||||
|
|
||||||
@@ -1348,8 +1311,7 @@ class TestSharedReservation:
|
|||||||
|
|
||||||
def test_different_groups_may_differ(self, sharing_server):
|
def test_different_groups_may_differ(self, sharing_server):
|
||||||
"""Only the members of ONE group have to agree; worker 0 and worker 1 of
|
"""Only the members of ONE group have to agree; worker 0 and worker 1 of
|
||||||
the same deployment hold different slices and never share a slot.
|
the same deployment hold different slices and never share a slot."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||||
server.reserve_serve_buffer(1, 1024, "plan-b")
|
server.reserve_serve_buffer(1, 1024, "plan-b")
|
||||||
@@ -1358,8 +1320,7 @@ class TestSharedReservation:
|
|||||||
|
|
||||||
def test_a_missing_digest_skips_the_check(self, sharing_server):
|
def test_a_missing_digest_skips_the_check(self, sharing_server):
|
||||||
"""The digest is optional on the wire, so an engine that does not send
|
"""The digest is optional on the wire, so an engine that does not send
|
||||||
one still reserves.
|
one still reserves."""
|
||||||
"""
|
|
||||||
server = sharing_server(workers_per_replica=2)
|
server = sharing_server(workers_per_replica=2)
|
||||||
server.reserve_serve_buffer(0, 1024, None)
|
server.reserve_serve_buffer(0, 1024, None)
|
||||||
server.reserve_serve_buffer(2, 1024, None)
|
server.reserve_serve_buffer(2, 1024, None)
|
||||||
|
|||||||
@@ -34,8 +34,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
|||||||
|
|
||||||
class _ListSource(WeightSource):
|
class _ListSource(WeightSource):
|
||||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||||
sharded-RDT group/order logic can be tested without a real model.
|
sharded-RDT group/order logic can be tested without a real model."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, pairs):
|
def __init__(self, pairs):
|
||||||
self._pairs = list(pairs)
|
self._pairs = list(pairs)
|
||||||
@@ -55,8 +54,7 @@ class _FakeProducerServer:
|
|||||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||||
groups flow back only through wait_freed / end_sync (see
|
groups flow back only through wait_freed / end_sync (see
|
||||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).
|
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, auto_free=True):
|
def __init__(self, auto_free=True):
|
||||||
self.order: list[str] = []
|
self.order: list[str] = []
|
||||||
@@ -100,8 +98,7 @@ class _FakeProducerServer:
|
|||||||
must already have a banked credit when the gate asks (auto_free, or a
|
must already have a banked credit when the gate asks (auto_free, or a
|
||||||
test's own free_group) — anything else is the deadlock the real
|
test's own free_group) — anything else is the deadlock the real
|
||||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||||
pacing, not a lifecycle milestone.
|
pacing, not a lifecycle milestone."""
|
||||||
"""
|
|
||||||
assert self._pending_freed, (
|
assert self._pending_freed, (
|
||||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||||
)
|
)
|
||||||
@@ -124,8 +121,7 @@ def _rdt_engine_with_fake_server(
|
|||||||
):
|
):
|
||||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||||
group-major metadata, and route _rpc to the fake.
|
group-major metadata, and route _rpc to the fake."""
|
||||||
"""
|
|
||||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||||
|
|
||||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||||
@@ -325,8 +321,7 @@ def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
|||||||
|
|
||||||
class _OwnedSource(_ListSource):
|
class _OwnedSource(_ListSource):
|
||||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||||
Takes group indices for convenience and declares the names inside them.
|
Takes group indices for convenience and declares the names inside them."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, pairs, owned_group_idx):
|
def __init__(self, pairs, owned_group_idx):
|
||||||
super().__init__(pairs)
|
super().__init__(pairs)
|
||||||
@@ -392,8 +387,7 @@ def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
|||||||
)
|
)
|
||||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||||
"""The free barrier's target is one integer per sync: the live consumer
|
"""The free barrier's target is one integer per sync: the live consumer
|
||||||
count, defaulting to the whole provisioned fleet.
|
count, defaulting to the whole provisioned fleet."""
|
||||||
"""
|
|
||||||
server = _FakeProducerServer(auto_free=True)
|
server = _FakeProducerServer(auto_free=True)
|
||||||
engine = _rdt_engine_with_fake_server(
|
engine = _rdt_engine_with_fake_server(
|
||||||
_rdt_source_two_layers(),
|
_rdt_source_two_layers(),
|
||||||
@@ -411,8 +405,7 @@ def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
|||||||
class TestLiveCountPlumbing:
|
class TestLiveCountPlumbing:
|
||||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||||
narrows each slot-sharing group's rendezvous.
|
narrows each slot-sharing group's rendezvous."""
|
||||||
"""
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _engine(num_consumers, world=2, rank=0):
|
def _engine(num_consumers, world=2, rank=0):
|
||||||
@@ -445,8 +438,7 @@ class TestLiveCountPlumbing:
|
|||||||
|
|
||||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||||
"""The whole point of the barrier: no routed per-producer targets, so
|
"""The whole point of the barrier: no routed per-producer targets, so
|
||||||
the identity of the dead consumer is irrelevant to the producers.
|
the identity of the dead consumer is irrelevant to the producers."""
|
||||||
"""
|
|
||||||
counts = []
|
counts = []
|
||||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||||
engine, got = self._engine(num_consumers=8)
|
engine, got = self._engine(num_consumers=8)
|
||||||
@@ -457,8 +449,7 @@ class TestLiveCountPlumbing:
|
|||||||
def test_the_live_ids_travel_with_the_count(self):
|
def test_the_live_ids_travel_with_the_count(self):
|
||||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||||
is still waiting for. They must describe the same set.
|
is still waiting for. They must describe the same set."""
|
||||||
"""
|
|
||||||
engine, got = self._engine(num_consumers=8)
|
engine, got = self._engine(num_consumers=8)
|
||||||
engine.send_weights([6, 0, 2, 2])
|
engine.send_weights([6, 0, 2, 2])
|
||||||
((count, ids),) = got
|
((count, ids),) = got
|
||||||
@@ -526,8 +517,7 @@ def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
|||||||
|
|
||||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||||
"""Every name must be held somewhere or it can never be served — caught
|
"""Every name must be held somewhere or it can never be served — caught
|
||||||
when the holdings are transposed, naming the orphan.
|
when the holdings are transposed, naming the orphan."""
|
||||||
"""
|
|
||||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||||
_rdt_engine_with_fake_server(
|
_rdt_engine_with_fake_server(
|
||||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||||
@@ -541,8 +531,7 @@ def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
|||||||
|
|
||||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||||
just its own share must fail loudly rather than silently drop the rest.
|
just its own share must fail loudly rather than silently drop the rest."""
|
||||||
"""
|
|
||||||
engine = _rdt_engine_with_fake_server(
|
engine = _rdt_engine_with_fake_server(
|
||||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||||
is_sender=False,
|
is_sender=False,
|
||||||
@@ -565,8 +554,7 @@ def _serve_ring_server(src_name, src):
|
|||||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||||
what puts the destination-view cache, and only it, under test.
|
what puts the destination-view cache, and only it, under test."""
|
||||||
"""
|
|
||||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||||
_RDTProducerServer,
|
_RDTProducerServer,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -419,8 +419,7 @@ def test_tensor_broadcast():
|
|||||||
|
|
||||||
def _dumps_oob(obj) -> tuple[bytes, list]:
|
def _dumps_oob(obj) -> tuple[bytes, list]:
|
||||||
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
|
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
|
||||||
dispatch table + out-of-band buffers >= 1MiB.
|
dispatch table + out-of-band buffers >= 1MiB."""
|
||||||
"""
|
|
||||||
buffers = []
|
buffers = []
|
||||||
|
|
||||||
def callback(buf: pickle.PickleBuffer) -> bool:
|
def callback(buf: pickle.PickleBuffer) -> bool:
|
||||||
@@ -491,8 +490,7 @@ def test_tensor_pickle_roundtrip(case: str):
|
|||||||
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
|
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
|
||||||
def test_reduce_tensor_fallback(case: str):
|
def test_reduce_tensor_fallback(case: str):
|
||||||
"""Tensors the zero-copy reducer can't safely alias must fall back to
|
"""Tensors the zero-copy reducer can't safely alias must fall back to
|
||||||
torch's default reduction.
|
torch's default reduction."""
|
||||||
"""
|
|
||||||
if case == "cuda":
|
if case == "cuda":
|
||||||
if not torch.cuda.is_available():
|
if not torch.cuda.is_available():
|
||||||
pytest.skip("requires CUDA")
|
pytest.skip("requires CUDA")
|
||||||
@@ -755,8 +753,7 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
|
|||||||
Pre-fix, the writer probed a port with get_open_port() and bound it
|
Pre-fix, the writer probed a port with get_open_port() and bound it
|
||||||
afterwards; pinning the probe to one free port makes every writer
|
afterwards; pinning the probe to one free port makes every writer
|
||||||
bind the same port and fail deterministically on that code path,
|
bind the same port and fail deterministically on that code path,
|
||||||
while the late-binding implementation never consults the probe.
|
while the late-binding implementation never consults the probe."""
|
||||||
"""
|
|
||||||
from vllm.distributed.device_communicators import shm_broadcast
|
from vllm.distributed.device_communicators import shm_broadcast
|
||||||
|
|
||||||
colliding_port = get_open_port()
|
colliding_port = get_open_port()
|
||||||
|
|||||||
@@ -31,8 +31,7 @@ class _CUDADeviceCountStatelessTestActor:
|
|||||||
|
|
||||||
def test_cuda_device_count_stateless():
|
def test_cuda_device_count_stateless():
|
||||||
"""Test that cuda_device_count_stateless changes return value if
|
"""Test that cuda_device_count_stateless changes return value if
|
||||||
CUDA_VISIBLE_DEVICES is changed.
|
CUDA_VISIBLE_DEVICES is changed."""
|
||||||
"""
|
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
|
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
|
||||||
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
|
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
|
||||||
|
|||||||
@@ -1166,8 +1166,7 @@ class RecordingClient:
|
|||||||
|
|
||||||
def _module_with(*pairs):
|
def _module_with(*pairs):
|
||||||
"""A tiny nn.Module exposing the given (name, tensor) pairs as parameters,
|
"""A tiny nn.Module exposing the given (name, tensor) pairs as parameters,
|
||||||
so trainer tests can build a ModuleSource without a real model.
|
so trainer tests can build a ModuleSource without a real model."""
|
||||||
"""
|
|
||||||
module = torch.nn.Module()
|
module = torch.nn.Module()
|
||||||
for name, tensor in pairs:
|
for name, tensor in pairs:
|
||||||
module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False))
|
module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False))
|
||||||
@@ -1288,8 +1287,7 @@ class TestModuleSource:
|
|||||||
|
|
||||||
def test_metadata_agrees_with_iteration(self):
|
def test_metadata_agrees_with_iteration(self):
|
||||||
"""The two channels must line up element-for-element: engines declare
|
"""The two channels must line up element-for-element: engines declare
|
||||||
the round from `metadata()` and then send what iteration yields.
|
the round from `metadata()` and then send what iteration yields."""
|
||||||
"""
|
|
||||||
source = ModuleSource(
|
source = ModuleSource(
|
||||||
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
|
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
|
||||||
)
|
)
|
||||||
@@ -1303,13 +1301,11 @@ class TestModuleSource:
|
|||||||
class TestWeightSourceGroupContract:
|
class TestWeightSourceGroupContract:
|
||||||
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
|
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
|
||||||
backends gather and free by, so the default must agree with
|
backends gather and free by, so the default must agree with
|
||||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds.
|
`layerwise_groups` over `metadata()`, restricted to what this rank holds."""
|
||||||
"""
|
|
||||||
|
|
||||||
class _Source(WeightSource):
|
class _Source(WeightSource):
|
||||||
"""Minimal source over an ordered (name, tensor) list, optionally holding
|
"""Minimal source over an ordered (name, tensor) list, optionally holding
|
||||||
only some names (in which case it iterates only their groups).
|
only some names (in which case it iterates only their groups)."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, names, held=None, reverse=False):
|
def __init__(self, names, held=None, reverse=False):
|
||||||
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
|
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
|
||||||
@@ -1364,8 +1360,7 @@ class TestWeightSourceGroupContract:
|
|||||||
def test_groups_order_follows_metadata_not_the_declaration(self):
|
def test_groups_order_follows_metadata_not_the_declaration(self):
|
||||||
"""The declaration is a SET of names; the groups it selects still come
|
"""The declaration is a SET of names; the groups it selects still come
|
||||||
out in metadata order, so each pairs with the right ``iter_groups()``
|
out in metadata order, so each pairs with the right ``iter_groups()``
|
||||||
batch however the source listed them.
|
batch however the source listed them."""
|
||||||
"""
|
|
||||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||||
source = self._source(
|
source = self._source(
|
||||||
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
|
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
|
||||||
@@ -1399,16 +1394,14 @@ class TestWeightSourceGroupContract:
|
|||||||
|
|
||||||
def test_out_of_order_iteration_raises(self):
|
def test_out_of_order_iteration_raises(self):
|
||||||
"""Materializing is usually a collective, so a rank that iterates out of
|
"""Materializing is usually a collective, so a rank that iterates out of
|
||||||
order deadlocks its peers -- fail loudly instead.
|
order deadlocks its peers -- fail loudly instead."""
|
||||||
"""
|
|
||||||
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
|
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
|
||||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||||
list(source.iter_groups())
|
list(source.iter_groups())
|
||||||
|
|
||||||
def test_a_source_may_override_iter_groups(self):
|
def test_a_source_may_override_iter_groups(self):
|
||||||
"""The extension point: materialize a whole group in one step instead of
|
"""The extension point: materialize a whole group in one step instead of
|
||||||
one generator resume per tensor.
|
one generator resume per tensor."""
|
||||||
"""
|
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
class _Batched(TestWeightSourceGroupContract._Source):
|
class _Batched(TestWeightSourceGroupContract._Source):
|
||||||
@@ -1429,8 +1422,7 @@ class TestDeferredProcessingContract:
|
|||||||
caller that takes over the update tail (running its own
|
caller that takes over the update tail (running its own
|
||||||
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
|
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
|
||||||
reads the flag and calls the method. Both must be answerable on any engine, or
|
reads the flag and calls the method. Both must be answerable on any engine, or
|
||||||
that caller ends up reaching through a getattr.
|
that caller ends up reaching through a getattr."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _engines(self):
|
def _engines(self):
|
||||||
return {
|
return {
|
||||||
@@ -1444,8 +1436,7 @@ class TestDeferredProcessingContract:
|
|||||||
|
|
||||||
def test_every_engine_can_be_drained(self):
|
def test_every_engine_can_be_drained(self):
|
||||||
"""The default is a no-op, so a caller never has to check whether the
|
"""The default is a no-op, so a caller never has to check whether the
|
||||||
method exists before calling it.
|
method exists before calling it."""
|
||||||
"""
|
|
||||||
for name, cls in self._engines().items():
|
for name, cls in self._engines().items():
|
||||||
assert callable(cls.drain_pending), name
|
assert callable(cls.drain_pending), name
|
||||||
|
|
||||||
@@ -1539,8 +1530,7 @@ class TestTrainerEngineBase:
|
|||||||
)
|
)
|
||||||
def test_ipc_trainer_send_weights_drives_client_in_order():
|
def test_ipc_trainer_send_weights_drives_client_in_order():
|
||||||
"""send_weights issues start -> update -> finish and ships per-round metadata;
|
"""send_weights issues start -> update -> finish and ships per-round metadata;
|
||||||
the packed wire param rides the init info, not the per-round update_info.
|
the packed wire param rides the init info, not the per-round update_info."""
|
||||||
"""
|
|
||||||
client = RecordingClient()
|
client = RecordingClient()
|
||||||
engine = IPCTrainerWeightTransferEngine(
|
engine = IPCTrainerWeightTransferEngine(
|
||||||
client=client,
|
client=client,
|
||||||
@@ -1559,8 +1549,7 @@ def test_ipc_trainer_send_weights_drives_client_in_order():
|
|||||||
|
|
||||||
def test_ipc_trainer_init_ships_packed_to_worker():
|
def test_ipc_trainer_init_ships_packed_to_worker():
|
||||||
"""trainer_init drives the inference-side init handshake and propagates the
|
"""trainer_init drives the inference-side init handshake and propagates the
|
||||||
must-agree `packed` flag to the worker.
|
must-agree `packed` flag to the worker."""
|
||||||
"""
|
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU (CUDA IPC handles).")
|
pytest.skip("Need at least 1 GPU (CUDA IPC handles).")
|
||||||
|
|
||||||
@@ -1581,8 +1570,7 @@ def test_ipc_trainer_init_ships_packed_to_worker():
|
|||||||
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||||
"""The sender's trainer_init drives the inference-side init handshake with
|
"""The sender's trainer_init drives the inference-side init handshake with
|
||||||
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
|
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
|
||||||
and propagates the must-agree wire params to the worker.
|
and propagates the must-agree wire params to the worker."""
|
||||||
"""
|
|
||||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||||
|
|
||||||
# Bypass the real NCCL rendezvous.
|
# Bypass the real NCCL rendezvous.
|
||||||
@@ -1623,8 +1611,7 @@ def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
|||||||
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||||
"""The worker engine reads packed + buffer geometry from the
|
"""The worker engine reads packed + buffer geometry from the
|
||||||
trainer-supplied init info at the handshake, not from the config or the
|
trainer-supplied init info at the handshake, not from the config or the
|
||||||
per-round update info.
|
per-round update info."""
|
||||||
"""
|
|
||||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@@ -1657,8 +1644,7 @@ def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
|||||||
|
|
||||||
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||||
"""Non-sender trainer ranks build an engine without opening an endpoint or
|
"""Non-sender trainer ranks build an engine without opening an endpoint or
|
||||||
touching the client; they only join the collectives in send_weights.
|
touching the client; they only join the collectives in send_weights."""
|
||||||
"""
|
|
||||||
client = RecordingClient()
|
client = RecordingClient()
|
||||||
engine = WeightTransferTrainerFactory.trainer_init(
|
engine = WeightTransferTrainerFactory.trainer_init(
|
||||||
init_info=NCCLTrainerInitInfo(
|
init_info=NCCLTrainerInitInfo(
|
||||||
@@ -1688,8 +1674,7 @@ def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
|||||||
def test_nccl_trainer_send_weights_drives_client_in_order():
|
def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||||
"""send_weights issues start -> update -> finish and ships per-round
|
"""send_weights issues start -> update -> finish and ships per-round
|
||||||
metadata; the packed wire params ride the init handshake, not the
|
metadata; the packed wire params ride the init handshake, not the
|
||||||
per-round update_info.
|
per-round update_info."""
|
||||||
"""
|
|
||||||
client = RecordingClient()
|
client = RecordingClient()
|
||||||
engine = NCCLTrainerWeightTransferEngine(
|
engine = NCCLTrainerWeightTransferEngine(
|
||||||
client=client,
|
client=client,
|
||||||
@@ -1710,8 +1695,7 @@ def test_nccl_trainer_send_weights_drives_client_in_order():
|
|||||||
|
|
||||||
class _ScriptedSource(WeightSource):
|
class _ScriptedSource(WeightSource):
|
||||||
"""Declares `meta` but yields whatever `pairs` says — used to drive the
|
"""Declares `meta` but yields whatever `pairs` says — used to drive the
|
||||||
metadata/iteration agreement checks.
|
metadata/iteration agreement checks."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, meta, pairs):
|
def __init__(self, meta, pairs):
|
||||||
self._meta = meta
|
self._meta = meta
|
||||||
@@ -1747,8 +1731,7 @@ def test_nccl_trainer_init_requires_source():
|
|||||||
|
|
||||||
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||||
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
|
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
|
||||||
iteration disagreeing with it must raise rather than corrupt the stream.
|
iteration disagreeing with it must raise rather than corrupt the stream."""
|
||||||
"""
|
|
||||||
meta = [
|
meta = [
|
||||||
ParamMeta("w", torch.float32, (4,)),
|
ParamMeta("w", torch.float32, (4,)),
|
||||||
ParamMeta("b", torch.float32, (2,)),
|
ParamMeta("b", torch.float32, (2,)),
|
||||||
@@ -1762,8 +1745,7 @@ def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
|||||||
|
|
||||||
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
|
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
|
||||||
"""A source that declares one wire dtype and materializes another would make
|
"""A source that declares one wire dtype and materializes another would make
|
||||||
the two sides disagree on every byte offset.
|
the two sides disagree on every byte offset."""
|
||||||
"""
|
|
||||||
meta = [ParamMeta("w", torch.float32, (4,))]
|
meta = [ParamMeta("w", torch.float32, (4,))]
|
||||||
engine = _mock_group_engine(
|
engine = _mock_group_engine(
|
||||||
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
|
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
|
||||||
@@ -1790,8 +1772,7 @@ def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
|
|||||||
|
|
||||||
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
|
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
|
||||||
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
|
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
|
||||||
be linearized first or the worker receives unrelated memory.
|
be linearized first or the worker receives unrelated memory."""
|
||||||
"""
|
|
||||||
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||||
view = base.t() # non-contiguous
|
view = base.t() # non-contiguous
|
||||||
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
|
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
|
||||||
@@ -1861,8 +1842,7 @@ def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
|
|||||||
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||||
"""The sender's trainer_init drives the init handshake with the
|
"""The sender's trainer_init drives the init handshake with the
|
||||||
worker-shaped init info; sparse ships no packed wire params, so the worker
|
worker-shaped init info; sparse ships no packed wire params, so the worker
|
||||||
keeps its unpacked defaults. Sparse takes no `source`.
|
keeps its unpacked defaults. Sparse takes no `source`."""
|
||||||
"""
|
|
||||||
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
|
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
|
||||||
|
|
||||||
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
|
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
|
||||||
@@ -1915,8 +1895,7 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
|||||||
|
|
||||||
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||||
"""A round with no patches must not touch the client (an empty sparse
|
"""A round with no patches must not touch the client (an empty sparse
|
||||||
update info is invalid by construction).
|
update info is invalid by construction)."""
|
||||||
"""
|
|
||||||
client = RecordingClient()
|
client = RecordingClient()
|
||||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||||
engine.model_update_group = MagicMock()
|
engine.model_update_group = MagicMock()
|
||||||
@@ -1952,8 +1931,7 @@ def test_sparse_nccl_trainer_rejects_source():
|
|||||||
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
||||||
"""Malformed patches must fail on the trainer, before start_weight_update:
|
"""Malformed patches must fail on the trainer, before start_weight_update:
|
||||||
the worker's own checks only run once the broadcasts are already in flight,
|
the worker's own checks only run once the broadcasts are already in flight,
|
||||||
where a size mismatch wedges both sides instead of raising.
|
where a size mismatch wedges both sides instead of raising."""
|
||||||
"""
|
|
||||||
client = RecordingClient()
|
client = RecordingClient()
|
||||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||||
engine.model_update_group = MagicMock()
|
engine.model_update_group = MagicMock()
|
||||||
|
|||||||
@@ -403,8 +403,7 @@ def trainer_broadcast_tensor_torch_free(
|
|||||||
def _run_uid_transfer(trainer_task) -> None:
|
def _run_uid_transfer(trainer_task) -> None:
|
||||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||||
and assert the tensor round-trips. Both ranks must enter init together --
|
and assert the tensor round-trips. Both ranks must enter init together --
|
||||||
there is no store barrier on the UID path.
|
there is no store barrier on the UID path."""
|
||||||
"""
|
|
||||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||||
|
|
||||||
_init_ray_for_weight_transfer()
|
_init_ray_for_weight_transfer()
|
||||||
@@ -451,8 +450,7 @@ def _run_uid_transfer(trainer_task) -> None:
|
|||||||
)
|
)
|
||||||
def test_nccl_weight_transfer_between_processes_uid():
|
def test_nccl_weight_transfer_between_processes_uid():
|
||||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||||
ranks on vLLM's ``PyNcclCommunicator``.
|
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||||
"""
|
|
||||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||||
|
|
||||||
|
|
||||||
@@ -462,7 +460,6 @@ def test_nccl_weight_transfer_between_processes_uid():
|
|||||||
)
|
)
|
||||||
def test_nccl_weight_transfer_torch_free_trainer():
|
def test_nccl_weight_transfer_torch_free_trainer():
|
||||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||||
communicator, only a cupy buffer and raw ``ncclBroadcast``.
|
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||||
"""
|
|
||||||
pytest.importorskip("cupy")
|
pytest.importorskip("cupy")
|
||||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||||
|
|||||||
@@ -570,8 +570,7 @@ def test_attention_config():
|
|||||||
def test_multi_node_world_size_includes_pcp(monkeypatch):
|
def test_multi_node_world_size_includes_pcp(monkeypatch):
|
||||||
"""PCP expands the process world size, so the --nnodes divisibility check
|
"""PCP expands the process world size, so the --nnodes divisibility check
|
||||||
must include it. Without this, TP=1/PCP=2 over 2 nodes computes a world
|
must include it. Without this, TP=1/PCP=2 over 2 nodes computes a world
|
||||||
size of 1 and the launch is rejected before the engine starts.
|
size of 1 and the launch is rejected before the engine starts."""
|
||||||
"""
|
|
||||||
import vllm.config.vllm
|
import vllm.config.vllm
|
||||||
|
|
||||||
# PCP requires the V2 model runner, which is gated on Triton.
|
# PCP requires the V2 model runner, which is gated on Triton.
|
||||||
@@ -797,8 +796,7 @@ def test_expand_json_human_readable_numbers(input_json, expected_json):
|
|||||||
)
|
)
|
||||||
def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
|
def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
|
||||||
"""Cloud storage URIs should not be passed to get_model_path()
|
"""Cloud storage URIs should not be passed to get_model_path()
|
||||||
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs.
|
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs."""
|
||||||
"""
|
|
||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
|
|
||||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
||||||
@@ -810,8 +808,7 @@ def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
|
|||||||
|
|
||||||
def test_cloud_storage_tokenizer_skips_get_model_path(monkeypatch):
|
def test_cloud_storage_tokenizer_skips_get_model_path(monkeypatch):
|
||||||
"""Cloud storage tokenizer URI should not be passed to
|
"""Cloud storage tokenizer URI should not be passed to
|
||||||
get_model_path() when HF_HUB_OFFLINE=1.
|
get_model_path() when HF_HUB_OFFLINE=1."""
|
||||||
"""
|
|
||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
|
|
||||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
||||||
@@ -973,8 +970,7 @@ class TestDpDeviceIdSharding:
|
|||||||
def test_dp_supervisor_device_ids_stay_env_relative(self):
|
def test_dp_supervisor_device_ids_stay_env_relative(self):
|
||||||
"""Regression test: the DP supervisor must pass env-relative indices,
|
"""Regression test: the DP supervisor must pass env-relative indices,
|
||||||
not physical IDs, because each child re-resolves --device-ids
|
not physical IDs, because each child re-resolves --device-ids
|
||||||
against its inherited device-control env var.
|
against its inherited device-control env var."""
|
||||||
"""
|
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
from vllm.entrypoints.launchers.dp_supervisor import _build_device_ids
|
from vllm.entrypoints.launchers.dp_supervisor import _build_device_ids
|
||||||
@@ -1001,8 +997,7 @@ class TestDpDeviceIdSharding:
|
|||||||
|
|
||||||
def test_dp_rank_shards_user_assigned_gpu_ids(self):
|
def test_dp_rank_shards_user_assigned_gpu_ids(self):
|
||||||
"""get_physical_gpu_ids_for_local_dp_rank slices the user-provided
|
"""get_physical_gpu_ids_for_local_dp_rank slices the user-provided
|
||||||
--device-ids list instead of recomputing from the env var.
|
--device-ids list instead of recomputing from the env var."""
|
||||||
"""
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.v1.engine.utils import get_physical_gpu_ids_for_local_dp_rank
|
from vllm.v1.engine.utils import get_physical_gpu_ids_for_local_dp_rank
|
||||||
|
|
||||||
|
|||||||
@@ -241,8 +241,7 @@ class TestToolResultContent:
|
|||||||
self, tool_result_content
|
self, tool_result_content
|
||||||
) -> AnthropicMessagesRequest:
|
) -> AnthropicMessagesRequest:
|
||||||
"""Build a request with assistant tool_use followed by user
|
"""Build a request with assistant tool_use followed by user
|
||||||
tool_result.
|
tool_result."""
|
||||||
"""
|
|
||||||
return _make_request(
|
return _make_request(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
@@ -327,8 +326,7 @@ class TestToolResultContent:
|
|||||||
|
|
||||||
def test_tool_result_with_text_and_image(self):
|
def test_tool_result_with_text_and_image(self):
|
||||||
"""Mixed text+image tool_result: text in tool msg, image in user
|
"""Mixed text+image tool_result: text in tool msg, image in user
|
||||||
msg.
|
msg."""
|
||||||
"""
|
|
||||||
request = self._make_tool_result_request(
|
request = self._make_tool_result_request(
|
||||||
[
|
[
|
||||||
{"type": "text", "text": "Here is the screenshot"},
|
{"type": "text", "text": "Here is the screenshot"},
|
||||||
@@ -425,8 +423,7 @@ class TestToolResultContent:
|
|||||||
class TestAttributionHeaderStripping:
|
class TestAttributionHeaderStripping:
|
||||||
def test_billing_header_stripped_from_system(self):
|
def test_billing_header_stripped_from_system(self):
|
||||||
"""Claude Code's x-anthropic-billing-header block should be
|
"""Claude Code's x-anthropic-billing-header block should be
|
||||||
stripped to preserve prefix caching.
|
stripped to preserve prefix caching."""
|
||||||
"""
|
|
||||||
request = _make_request(
|
request = _make_request(
|
||||||
[{"role": "user", "content": "Hello"}],
|
[{"role": "user", "content": "Hello"}],
|
||||||
system=[
|
system=[
|
||||||
@@ -1150,8 +1147,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
|||||||
async def test_buffered_content_flushed_on_done_without_usage_chunk(self):
|
async def test_buffered_content_flushed_on_done_without_usage_chunk(self):
|
||||||
"""Content buffered during tool_use must be emitted even if the
|
"""Content buffered during tool_use must be emitted even if the
|
||||||
stream jumps straight from finish_reason to [DONE], skipping the
|
stream jumps straight from finish_reason to [DONE], skipping the
|
||||||
empty-choices usage chunk.
|
empty-choices usage chunk."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def sse_input():
|
async def sse_input():
|
||||||
yield _make_stream_chunk(
|
yield _make_stream_chunk(
|
||||||
@@ -1255,8 +1251,7 @@ class TestStreamingCacheUsageSemantics:
|
|||||||
async def test_streaming_cache_fields_absent_then_populated(self):
|
async def test_streaming_cache_fields_absent_then_populated(self):
|
||||||
"""First chunk lacks prompt_tokens_details (vLLM contract);
|
"""First chunk lacks prompt_tokens_details (vLLM contract);
|
||||||
message_start omits cache fields. The final chunk carries
|
message_start omits cache fields. The final chunk carries
|
||||||
prompt_tokens_details, so message_delta carries resolved values.
|
prompt_tokens_details, so message_delta carries resolved values."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def sse_input():
|
async def sse_input():
|
||||||
yield _make_stream_chunk(
|
yield _make_stream_chunk(
|
||||||
@@ -1301,8 +1296,7 @@ class TestStreamingCacheUsageSemantics:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_streaming_no_cache_hit(self):
|
async def test_streaming_no_cache_hit(self):
|
||||||
"""When the final chunk reports cached_tokens=0, message_delta carries
|
"""When the final chunk reports cached_tokens=0, message_delta carries
|
||||||
cache fields = 0 (cache miss); message_start still omits them.
|
cache fields = 0 (cache miss); message_start still omits them."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def sse_input():
|
async def sse_input():
|
||||||
yield _make_stream_chunk(
|
yield _make_stream_chunk(
|
||||||
@@ -1343,8 +1337,7 @@ class TestStreamingCacheUsageSemantics:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_streaming_no_prompt_tokens_details_at_all(self):
|
async def test_streaming_no_prompt_tokens_details_at_all(self):
|
||||||
"""If --enable-prompt-tokens-details is off, no chunk carries cache
|
"""If --enable-prompt-tokens-details is off, no chunk carries cache
|
||||||
info; both message_start and message_delta omit cache fields.
|
info; both message_start and message_delta omit cache fields."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def sse_input():
|
async def sse_input():
|
||||||
yield _make_stream_chunk(
|
yield _make_stream_chunk(
|
||||||
@@ -1465,8 +1458,7 @@ class TestMessagesFullConverter:
|
|||||||
class TestCacheSalt:
|
class TestCacheSalt:
|
||||||
def test_cache_salt_passed_through(self):
|
def test_cache_salt_passed_through(self):
|
||||||
"""cache_salt on the Anthropic request reaches the converted
|
"""cache_salt on the Anthropic request reaches the converted
|
||||||
ChatCompletionRequest so prefix-cache isolation works via /v1/messages.
|
ChatCompletionRequest so prefix-cache isolation works via /v1/messages."""
|
||||||
"""
|
|
||||||
request = _make_request(
|
request = _make_request(
|
||||||
[{"role": "user", "content": "Hello"}],
|
[{"role": "user", "content": "Hello"}],
|
||||||
cache_salt="tenant-abc-secret-salt",
|
cache_salt="tenant-abc-secret-salt",
|
||||||
@@ -1671,8 +1663,7 @@ class TestClientErrorResponses:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _conversion_error() -> ValidationError:
|
def _conversion_error() -> ValidationError:
|
||||||
"""A real pydantic ValidationError like the one ChatCompletionRequest
|
"""A real pydantic ValidationError like the one ChatCompletionRequest
|
||||||
construction raises when Anthropic input violates the OpenAI schema.
|
construction raises when Anthropic input violates the OpenAI schema."""
|
||||||
"""
|
|
||||||
|
|
||||||
class _StubRequest(BaseModel):
|
class _StubRequest(BaseModel):
|
||||||
stop: Annotated[list[str], Field(max_length=4)] | None = None
|
stop: Annotated[list[str], Field(max_length=4)] | None = None
|
||||||
@@ -1683,8 +1674,7 @@ class TestClientErrorResponses:
|
|||||||
|
|
||||||
def test_validation_error_returns_bad_request(self):
|
def test_validation_error_returns_bad_request(self):
|
||||||
"""A pydantic ValidationError during Anthropic->OpenAI conversion is
|
"""A pydantic ValidationError during Anthropic->OpenAI conversion is
|
||||||
surfaced as a 400 BadRequestError, not a 500.
|
surfaced as a 400 BadRequestError, not a 500."""
|
||||||
"""
|
|
||||||
handler = MagicMock(spec=AnthropicServingMessages)
|
handler = MagicMock(spec=AnthropicServingMessages)
|
||||||
handler.create_messages.side_effect = self._conversion_error()
|
handler.create_messages.side_effect = self._conversion_error()
|
||||||
|
|
||||||
|
|||||||
@@ -87,8 +87,7 @@ def _make_chunk(
|
|||||||
citations: list[Any] | None = None,
|
citations: list[Any] | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
|
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
|
||||||
consumes.
|
consumes."""
|
||||||
"""
|
|
||||||
delta: dict[str, Any] = {}
|
delta: dict[str, Any] = {}
|
||||||
if role is not None:
|
if role is not None:
|
||||||
delta["role"] = role
|
delta["role"] = role
|
||||||
@@ -136,8 +135,7 @@ async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
|
|||||||
|
|
||||||
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
|
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
|
||||||
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
||||||
the emitted SSE frames.
|
the emitted SSE frames."""
|
||||||
"""
|
|
||||||
request = CohereChatV2Request(
|
request = CohereChatV2Request(
|
||||||
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
|
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -50,8 +50,7 @@ def update_admission_stats_worker(listen_address, sock, args, client_config):
|
|||||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||||
endpoints back via the pipe, then exit.
|
endpoints back via the pipe, then exit."""
|
||||||
"""
|
|
||||||
ctx = zmq.Context()
|
ctx = zmq.Context()
|
||||||
try:
|
try:
|
||||||
in_sock = make_zmq_socket(
|
in_sock = make_zmq_socket(
|
||||||
@@ -358,8 +357,7 @@ def test_external_process_monitoring(api_server_args):
|
|||||||
def test_gather_actual_addresses_end_to_end():
|
def test_gather_actual_addresses_end_to_end():
|
||||||
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
|
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
|
||||||
the bound endpoints back via its per-child pipe; the manager surfaces
|
the bound endpoints back via its per-child pipe; the manager surfaces
|
||||||
them via :py:meth:`gather_actual_addresses`.
|
them via :py:meth:`gather_actual_addresses`."""
|
||||||
"""
|
|
||||||
host = "127.0.0.1"
|
host = "127.0.0.1"
|
||||||
num_servers = 4
|
num_servers = 4
|
||||||
|
|
||||||
@@ -409,8 +407,7 @@ def test_gather_actual_addresses_end_to_end():
|
|||||||
@pytest.mark.timeout(30)
|
@pytest.mark.timeout(30)
|
||||||
def test_gather_actual_addresses_child_crash_before_report():
|
def test_gather_actual_addresses_child_crash_before_report():
|
||||||
"""A child that exits before sending its endpoints must surface a
|
"""A child that exits before sending its endpoints must surface a
|
||||||
clear ``RuntimeError`` rather than hang or return ``None`` slots.
|
clear ``RuntimeError`` rather than hang or return ``None`` slots."""
|
||||||
"""
|
|
||||||
host = "127.0.0.1"
|
host = "127.0.0.1"
|
||||||
num_servers = 2
|
num_servers = 2
|
||||||
placeholder_inputs = [
|
placeholder_inputs = [
|
||||||
@@ -447,8 +444,7 @@ def test_gather_actual_addresses_child_crash_before_report():
|
|||||||
|
|
||||||
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||||
"""The Rust frontend command carries every non-default arg as JSON.
|
"""The Rust frontend command carries every non-default arg as JSON.
|
||||||
Credentials must not reach the log line.
|
Credentials must not reach the log line."""
|
||||||
"""
|
|
||||||
import subprocess as subprocess_mod
|
import subprocess as subprocess_mod
|
||||||
|
|
||||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||||
|
|||||||
@@ -186,8 +186,7 @@ def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
|
|||||||
|
|
||||||
def test_passes_with_reasoning_parser(serve_parser):
|
def test_passes_with_reasoning_parser(serve_parser):
|
||||||
"""Ensure validation passes if reasoning is enabled
|
"""Ensure validation passes if reasoning is enabled
|
||||||
with a reasoning parser
|
with a reasoning parser"""
|
||||||
"""
|
|
||||||
args = serve_parser.parse_args(
|
args = serve_parser.parse_args(
|
||||||
args=[
|
args=[
|
||||||
"--reasoning-parser",
|
"--reasoning-parser",
|
||||||
@@ -382,8 +381,7 @@ def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
|||||||
)
|
)
|
||||||
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||||
"""Ensure that users don't misuse --served-model-name and end up with the default
|
"""Ensure that users don't misuse --served-model-name and end up with the default
|
||||||
model tag instead of the one they intended to serve.
|
model tag instead of the one they intended to serve."""
|
||||||
"""
|
|
||||||
# Call the serve subparser
|
# Call the serve subparser
|
||||||
args.insert(0, "serve")
|
args.insert(0, "serve")
|
||||||
# Create a dummy config file if the test case includes it
|
# Create a dummy config file if the test case includes it
|
||||||
|
|||||||
+1
-2
@@ -2,8 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
|
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
|
||||||
Chat Completions request.
|
Chat Completions request."""
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
|||||||
+1
-2
@@ -68,8 +68,7 @@ async def client(server):
|
|||||||
def image_url() -> str:
|
def image_url() -> str:
|
||||||
"""Stable real image as a data URL, kept identical across both the
|
"""Stable real image as a data URL, kept identical across both the
|
||||||
text and prompt_embeds requests so any output difference must come from
|
text and prompt_embeds requests so any output difference must come from
|
||||||
how the text content is delivered.
|
how the text content is delivered."""
|
||||||
"""
|
|
||||||
return encode_image_url(ImageAsset("stop_sign").pil_image)
|
return encode_image_url(ImageAsset("stop_sign").pil_image)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -145,8 +145,7 @@ def describe_image_messages(
|
|||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Build the system + user messages used by the completions-with-image
|
"""Build the system + user messages used by the completions-with-image
|
||||||
family of tests. *extra_image_fields* is merged into the top-level
|
family of tests. *extra_image_fields* is merged into the top-level
|
||||||
image content block (for uuid / bad-key tests).
|
image content block (for uuid / bad-key tests)."""
|
||||||
"""
|
|
||||||
image_block: dict = {
|
image_block: dict = {
|
||||||
"type": "image_url",
|
"type": "image_url",
|
||||||
"image_url": {"url": image_url},
|
"image_url": {"url": image_url},
|
||||||
@@ -176,8 +175,7 @@ async def complete_and_check(
|
|||||||
temperature: float = 0.0,
|
temperature: float = 0.0,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Run a chat completion and assert the output is non-empty.
|
"""Run a chat completion and assert the output is non-empty.
|
||||||
Returns the content string.
|
Returns the content string."""
|
||||||
"""
|
|
||||||
chat_completion = await client.chat.completions.create(
|
chat_completion = await client.chat.completions.create(
|
||||||
model=model_name,
|
model=model_name,
|
||||||
messages=messages,
|
messages=messages,
|
||||||
|
|||||||
@@ -465,8 +465,7 @@ def test_system_message_warns_on_video(video_content):
|
|||||||
def test_json_schema_response_format_missing_schema():
|
def test_json_schema_response_format_missing_schema():
|
||||||
"""When response_format type is 'json_schema' but the json_schema field
|
"""When response_format type is 'json_schema' but the json_schema field
|
||||||
is not provided, request construction should raise a validation error
|
is not provided, request construction should raise a validation error
|
||||||
so the API returns 400 instead of 500.
|
so the API returns 400 instead of 500."""
|
||||||
"""
|
|
||||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||||
ChatCompletionRequest(
|
ChatCompletionRequest(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
@@ -520,8 +519,7 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
|
|||||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
||||||
def test_non_numeric_logprobs_rejected(field_name):
|
def test_non_numeric_logprobs_rejected(field_name):
|
||||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
|
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||||
"""
|
|
||||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||||
ChatCompletionRequest(
|
ChatCompletionRequest(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
|
|||||||
@@ -81,8 +81,7 @@ async def test_chat_logit_bias_invalid(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_chat_logit_bias_non_integer_key(client):
|
async def test_chat_logit_bias_non_integer_key(client):
|
||||||
"""Test that a non-integer logit_bias key is rejected with a clean,
|
"""Test that a non-integer logit_bias key is rejected with a clean,
|
||||||
informative error instead of a raw 'invalid literal for int()' message.
|
informative error instead of a raw 'invalid literal for int()' message."""
|
||||||
"""
|
|
||||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||||
await client.chat.completions.create(
|
await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
@@ -102,8 +101,7 @@ async def test_chat_logit_bias_non_integer_key(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_chat_logit_bias_non_numeric_value(client):
|
async def test_chat_logit_bias_non_numeric_value(client):
|
||||||
"""Test that a non-numeric logit_bias value is rejected with a message
|
"""Test that a non-numeric logit_bias value is rejected with a message
|
||||||
that names the specific offending token, not just a generic TypeError.
|
that names the specific offending token, not just a generic TypeError."""
|
||||||
"""
|
|
||||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||||
await client.chat.completions.create(
|
await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
@@ -122,8 +120,7 @@ async def test_chat_logit_bias_non_numeric_value(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_chat_logit_bias_multiple_non_integer_keys(client):
|
async def test_chat_logit_bias_multiple_non_integer_keys(client):
|
||||||
"""Test that ALL invalid logit_bias keys are reported together,
|
"""Test that ALL invalid logit_bias keys are reported together,
|
||||||
not just the first one encountered.
|
not just the first one encountered."""
|
||||||
"""
|
|
||||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||||
await client.chat.completions.create(
|
await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
|
|||||||
@@ -54,8 +54,7 @@ def stop_sign_image_url():
|
|||||||
async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI):
|
async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI):
|
||||||
"""En -> es translation through TranslateGemma's bundled chat template's
|
"""En -> es translation through TranslateGemma's bundled chat template's
|
||||||
``text`` content branch, which depends on ``source_lang_code`` /
|
``text`` content branch, which depends on ``source_lang_code`` /
|
||||||
``target_lang_code`` extra fields being preserved on the content part.
|
``target_lang_code`` extra fields being preserved on the content part."""
|
||||||
"""
|
|
||||||
completion = await client.chat.completions.create(
|
completion = await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
messages=[
|
messages=[
|
||||||
@@ -99,8 +98,7 @@ async def test_translategemma_image_extra_lang_code_fields(
|
|||||||
``source_lang_code`` / ``target_lang_code`` extras must survive parsing
|
``source_lang_code`` / ``target_lang_code`` extras must survive parsing
|
||||||
and end up alongside the ``{"type": "image"}`` placeholder so the
|
and end up alongside the ``{"type": "image"}`` placeholder so the
|
||||||
template's image branch (``content["type"] == 'image'``) renders the
|
template's image branch (``content["type"] == 'image'``) renders the
|
||||||
right OCR-translation prompt.
|
right OCR-translation prompt."""
|
||||||
"""
|
|
||||||
completion = await client.chat.completions.create(
|
completion = await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
messages=[
|
messages=[
|
||||||
|
|||||||
@@ -191,8 +191,7 @@ async def test_logprob_token_ids_stream_with_no_top_k(server):
|
|||||||
async def test_logprob_token_ids_default_behavior_unchanged(server):
|
async def test_logprob_token_ids_default_behavior_unchanged(server):
|
||||||
"""Without `logprob_token_ids`, the response carries the natural top-k
|
"""Without `logprob_token_ids`, the response carries the natural top-k
|
||||||
most-likely tokens. This guards against the new field accidentally
|
most-likely tokens. This guards against the new field accidentally
|
||||||
changing the default-path output.
|
changing the default-path output."""
|
||||||
"""
|
|
||||||
async with server.get_async_client() as client:
|
async with server.get_async_client() as client:
|
||||||
resp = await client.chat.completions.create(
|
resp = await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
|
|||||||
@@ -1406,8 +1406,7 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_serving_chat_data_parallel_rank_extraction():
|
async def test_serving_chat_data_parallel_rank_extraction():
|
||||||
"""Test that data_parallel_rank is properly extracted from header and
|
"""Test that data_parallel_rank is properly extracted from header and
|
||||||
passed to engine.
|
passed to engine."""
|
||||||
"""
|
|
||||||
mock_engine = MagicMock(spec=AsyncLLM)
|
mock_engine = MagicMock(spec=AsyncLLM)
|
||||||
mock_engine.errored = False
|
mock_engine.errored = False
|
||||||
mock_engine.model_config = MockModelConfig()
|
mock_engine.model_config = MockModelConfig()
|
||||||
@@ -1729,8 +1728,7 @@ class TestServingChatWithHarmony:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_system_message_without_tools(self, serving_chat, stream):
|
async def test_system_message_without_tools(self, serving_chat, stream):
|
||||||
"""Leading system message produces a developer message with
|
"""Leading system message produces a developer message with
|
||||||
DeveloperContent (# Instructions header).
|
DeveloperContent (# Instructions header)."""
|
||||||
"""
|
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": "You are a helpful assistant."},
|
{"role": "system", "content": "You are a helpful assistant."},
|
||||||
{"role": "user", "content": "Hello"},
|
{"role": "user", "content": "Hello"},
|
||||||
@@ -1752,8 +1750,7 @@ class TestServingChatWithHarmony:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
|
async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
|
||||||
"""Leading system message is folded into the developer message
|
"""Leading system message is folded into the developer message
|
||||||
alongside tool definitions.
|
alongside tool definitions."""
|
||||||
"""
|
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": "You are a helpful assistant."},
|
{"role": "system", "content": "You are a helpful assistant."},
|
||||||
{"role": "user", "content": "What's the weather?"},
|
{"role": "user", "content": "What's the weather?"},
|
||||||
@@ -2168,8 +2165,7 @@ class TestServingChatWithHarmony:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_tool_choice_validation_without_parser():
|
async def test_tool_choice_validation_without_parser():
|
||||||
"""Test that tool_choice='required' or named tool without tool_parser
|
"""Test that tool_choice='required' or named tool without tool_parser
|
||||||
returns an appropriate error message.
|
returns an appropriate error message."""
|
||||||
"""
|
|
||||||
mock_engine = MagicMock(spec=AsyncLLM)
|
mock_engine = MagicMock(spec=AsyncLLM)
|
||||||
mock_engine.errored = False
|
mock_engine.errored = False
|
||||||
mock_engine.model_config = MockModelConfig()
|
mock_engine.model_config = MockModelConfig()
|
||||||
|
|||||||
@@ -151,8 +151,7 @@ async def client(request, server, server_with_auto_reasoning_config):
|
|||||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||||
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||||
"""Test that mixed requests (some with thinking_token_budget, some without)
|
"""Test that mixed requests (some with thinking_token_budget, some without)
|
||||||
complete successfully without errors.
|
complete successfully without errors."""
|
||||||
"""
|
|
||||||
response_with_budget = await client.chat.completions.create(
|
response_with_budget = await client.chat.completions.create(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
messages=MESSAGES,
|
messages=MESSAGES,
|
||||||
|
|||||||
@@ -521,8 +521,7 @@ async def test_completion_error_stream():
|
|||||||
def test_json_schema_response_format_missing_schema():
|
def test_json_schema_response_format_missing_schema():
|
||||||
"""When response_format type is 'json_schema' but the json_schema field
|
"""When response_format type is 'json_schema' but the json_schema field
|
||||||
is not provided, request construction should raise a validation error
|
is not provided, request construction should raise a validation error
|
||||||
so the API returns 400 instead of 500.
|
so the API returns 400 instead of 500."""
|
||||||
"""
|
|
||||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||||
CompletionRequest(
|
CompletionRequest(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
@@ -585,8 +584,7 @@ def test_negative_prompt_token_ids_flat():
|
|||||||
def test_logprobs_minus_one_allowed():
|
def test_logprobs_minus_one_allowed():
|
||||||
"""logprobs=-1 means "return all logprobs". The sampling layer and the chat
|
"""logprobs=-1 means "return all logprobs". The sampling layer and the chat
|
||||||
top_logprobs / prompt_logprobs validators all accept -1, so the completion
|
top_logprobs / prompt_logprobs validators all accept -1, so the completion
|
||||||
logprobs validator must accept it too instead of rejecting it as negative.
|
logprobs validator must accept it too instead of rejecting it as negative."""
|
||||||
"""
|
|
||||||
request = CompletionRequest(
|
request = CompletionRequest(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
prompt="Test prompt",
|
prompt="Test prompt",
|
||||||
@@ -746,8 +744,7 @@ class TestCompletionPromptListLimit:
|
|||||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "logprobs"])
|
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "logprobs"])
|
||||||
def test_non_numeric_logprobs_rejected(field_name):
|
def test_non_numeric_logprobs_rejected(field_name):
|
||||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
|
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||||
"""
|
|
||||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||||
CompletionRequest(
|
CompletionRequest(
|
||||||
model=MODEL_NAME,
|
model=MODEL_NAME,
|
||||||
|
|||||||
@@ -43,8 +43,7 @@ def _system():
|
|||||||
|
|
||||||
class TestResponseInputToHarmonyRenderParity:
|
class TestResponseInputToHarmonyRenderParity:
|
||||||
"""Each test drives the same conversation through both APIs and asserts
|
"""Each test drives the same conversation through both APIs and asserts
|
||||||
identical Harmony messages and rendered token sequences.
|
identical Harmony messages and rendered token sequences."""
|
||||||
"""
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
# Single-message cases
|
# Single-message cases
|
||||||
@@ -52,8 +51,7 @@ class TestResponseInputToHarmonyRenderParity:
|
|||||||
|
|
||||||
def test_developer_message(self):
|
def test_developer_message(self):
|
||||||
"""Both APIs must render developer messages identically using
|
"""Both APIs must render developer messages identically using
|
||||||
DeveloperContent (with the '# Instructions' header).
|
DeveloperContent (with the '# Instructions' header)."""
|
||||||
"""
|
|
||||||
chat_msgs = parse_chat_input_to_harmony_message(
|
chat_msgs = parse_chat_input_to_harmony_message(
|
||||||
{"role": "developer", "content": "Be concise."}
|
{"role": "developer", "content": "Be concise."}
|
||||||
)
|
)
|
||||||
@@ -248,8 +246,7 @@ class TestResponseInputToHarmonyRenderParity:
|
|||||||
def test_reasoning_combined_with_function_call(self):
|
def test_reasoning_combined_with_function_call(self):
|
||||||
"""Chat API packs reasoning + tool_calls into one dict; responses API
|
"""Chat API packs reasoning + tool_calls into one dict; responses API
|
||||||
represents them as two separate items. Both must produce the same two
|
represents them as two separate items. Both must produce the same two
|
||||||
Harmony messages in the same order: analysis then commentary.
|
Harmony messages in the same order: analysis then commentary."""
|
||||||
"""
|
|
||||||
chat_msgs = parse_chat_input_to_harmony_message(
|
chat_msgs = parse_chat_input_to_harmony_message(
|
||||||
{
|
{
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
@@ -532,8 +529,7 @@ class TestResponseInputToHarmonyRenderParity:
|
|||||||
def test_completed_turns_drop_reasoning(self):
|
def test_completed_turns_drop_reasoning(self):
|
||||||
"""Validates that reasoning from completed turns is dropped, while
|
"""Validates that reasoning from completed turns is dropped, while
|
||||||
reasoning from the current in-progress tool-call turn is preserved
|
reasoning from the current in-progress tool-call turn is preserved
|
||||||
in both chat completions and responses previous_input_messages.
|
in both chat completions and responses previous_input_messages."""
|
||||||
"""
|
|
||||||
first_turn_reasoning = "FIRST_TURN_REASONING"
|
first_turn_reasoning = "FIRST_TURN_REASONING"
|
||||||
second_turn_reasoning = "SECOND_TURN_REASONING"
|
second_turn_reasoning = "SECOND_TURN_REASONING"
|
||||||
|
|
||||||
|
|||||||
@@ -323,8 +323,7 @@ class TestCommonParseInputToHarmonyMessage:
|
|||||||
|
|
||||||
def test_system_message(self, parse_function):
|
def test_system_message(self, parse_function):
|
||||||
"""Test parsing system messages, which are parsed into developer messages
|
"""Test parsing system messages, which are parsed into developer messages
|
||||||
with DeveloperContent.
|
with DeveloperContent."""
|
||||||
"""
|
|
||||||
chat_msg = {
|
chat_msg = {
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": "You are a helpful assistant",
|
"content": "You are a helpful assistant",
|
||||||
|
|||||||
@@ -12,8 +12,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
|||||||
|
|
||||||
def test_function_call_dict_converted_to_object():
|
def test_function_call_dict_converted_to_object():
|
||||||
"""Test that function_call dictionaries are correctly parsed into
|
"""Test that function_call dictionaries are correctly parsed into
|
||||||
ResponseFunctionToolCall objects.
|
ResponseFunctionToolCall objects."""
|
||||||
"""
|
|
||||||
# Create a request with function_call as dict
|
# Create a request with function_call as dict
|
||||||
request_data = {
|
request_data = {
|
||||||
"model": "gpt-oss",
|
"model": "gpt-oss",
|
||||||
@@ -330,8 +329,7 @@ def test_validator_handles_empty_iterator():
|
|||||||
|
|
||||||
def test_assistant_string_content_stays_easyinput():
|
def test_assistant_string_content_stays_easyinput():
|
||||||
"""EasyInput assistant message with plain string content is not
|
"""EasyInput assistant message with plain string content is not
|
||||||
coerced into a ResponseOutputMessage.
|
coerced into a ResponseOutputMessage."""
|
||||||
"""
|
|
||||||
request_data = {
|
request_data = {
|
||||||
"model": "test-model",
|
"model": "test-model",
|
||||||
"input": [
|
"input": [
|
||||||
@@ -353,8 +351,7 @@ def test_assistant_string_content_stays_easyinput():
|
|||||||
|
|
||||||
def test_assistant_output_style_content_coerced():
|
def test_assistant_output_style_content_coerced():
|
||||||
"""Assistant message whose content is output-message-shaped (list of
|
"""Assistant message whose content is output-message-shaped (list of
|
||||||
output_text items) should be coerced to ResponseOutputMessage.
|
output_text items) should be coerced to ResponseOutputMessage."""
|
||||||
"""
|
|
||||||
request_data = {
|
request_data = {
|
||||||
"model": "test-model",
|
"model": "test-model",
|
||||||
"input": [
|
"input": [
|
||||||
|
|||||||
@@ -67,8 +67,7 @@ class TestResponseInputToHarmonyMessage:
|
|||||||
|
|
||||||
def test_system_message(self):
|
def test_system_message(self):
|
||||||
"""System messages carry developer instructions and must be rendered
|
"""System messages carry developer instructions and must be rendered
|
||||||
as developer messages with DeveloperContent.
|
as developer messages with DeveloperContent."""
|
||||||
"""
|
|
||||||
msg = response_input_to_harmony(
|
msg = response_input_to_harmony(
|
||||||
{"type": "message", "role": "system", "content": "Be helpful."},
|
{"type": "message", "role": "system", "content": "Be helpful."},
|
||||||
prev_responses=[],
|
prev_responses=[],
|
||||||
@@ -90,8 +89,7 @@ class TestResponseInputToHarmonyMessage:
|
|||||||
|
|
||||||
def test_developer_message_gets_instructions_prefix(self):
|
def test_developer_message_gets_instructions_prefix(self):
|
||||||
"""Developer messages must use DeveloperContent which adds the
|
"""Developer messages must use DeveloperContent which adds the
|
||||||
'# Instructions' header the model was trained on.
|
'# Instructions' header the model was trained on."""
|
||||||
"""
|
|
||||||
msg = response_input_to_harmony(
|
msg = response_input_to_harmony(
|
||||||
{"type": "message", "role": "developer", "content": "Be concise."},
|
{"type": "message", "role": "developer", "content": "Be concise."},
|
||||||
prev_responses=[],
|
prev_responses=[],
|
||||||
@@ -121,8 +119,7 @@ class TestResponseInputToHarmonyMessage:
|
|||||||
|
|
||||||
def test_developer_message_array_content_concatenated(self):
|
def test_developer_message_array_content_concatenated(self):
|
||||||
"""Array content in developer messages is flattened and rendered
|
"""Array content in developer messages is flattened and rendered
|
||||||
via DeveloperContent with the '# Instructions' header.
|
via DeveloperContent with the '# Instructions' header."""
|
||||||
"""
|
|
||||||
msg = response_input_to_harmony(
|
msg = response_input_to_harmony(
|
||||||
{
|
{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
@@ -213,8 +210,7 @@ class TestResponseInputToHarmonyMessage:
|
|||||||
|
|
||||||
def test_function_call_output_uses_most_recent_matching_call(self):
|
def test_function_call_output_uses_most_recent_matching_call(self):
|
||||||
"""When multiple prev_responses share a call_id, the last one wins
|
"""When multiple prev_responses share a call_id, the last one wins
|
||||||
because the search is reversed.
|
because the search is reversed."""
|
||||||
"""
|
|
||||||
earlier = ResponseFunctionToolCall(
|
earlier = ResponseFunctionToolCall(
|
||||||
id="fc_old",
|
id="fc_old",
|
||||||
call_id="call_test",
|
call_id="call_test",
|
||||||
|
|||||||
@@ -839,13 +839,11 @@ class TestConstructChatMessagesCombinePolicy:
|
|||||||
|
|
||||||
class TestConstructInputMessagesInstructionsLeak:
|
class TestConstructInputMessagesInstructionsLeak:
|
||||||
"""Regression tests for #37697: instructions from a prior response
|
"""Regression tests for #37697: instructions from a prior response
|
||||||
should NOT leak through previous_response_id.
|
should NOT leak through previous_response_id."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_old_instructions_stripped_from_prev_msg(self):
|
def test_old_instructions_stripped_from_prev_msg(self):
|
||||||
"""System message in prev_msg must be dropped so the new request's
|
"""System message in prev_msg must be dropped so the new request's
|
||||||
instructions are the only system message in the conversation.
|
instructions are the only system message in the conversation."""
|
||||||
"""
|
|
||||||
prev = [
|
prev = [
|
||||||
{"role": "system", "content": "old instructions"},
|
{"role": "system", "content": "old instructions"},
|
||||||
{"role": "user", "content": "What is 2+2?"},
|
{"role": "user", "content": "What is 2+2?"},
|
||||||
@@ -862,8 +860,7 @@ class TestConstructInputMessagesInstructionsLeak:
|
|||||||
|
|
||||||
def test_no_instructions_in_new_request(self):
|
def test_no_instructions_in_new_request(self):
|
||||||
"""If the new request has no instructions, old ones should still
|
"""If the new request has no instructions, old ones should still
|
||||||
be stripped -- they must not carry over.
|
be stripped -- they must not carry over."""
|
||||||
"""
|
|
||||||
prev = [
|
prev = [
|
||||||
{"role": "system", "content": "old instructions"},
|
{"role": "system", "content": "old instructions"},
|
||||||
{"role": "user", "content": "Hi"},
|
{"role": "user", "content": "Hi"},
|
||||||
|
|||||||
@@ -1163,8 +1163,7 @@ class TestHarmonyPreambleStreaming:
|
|||||||
|
|
||||||
def test_preamble_done_emits_text_done_events(self) -> None:
|
def test_preamble_done_emits_text_done_events(self) -> None:
|
||||||
"""Completed preamble should emit text done + content_part done +
|
"""Completed preamble should emit text done + content_part done +
|
||||||
output_item done, same shape as final channel.
|
output_item done, same shape as final channel."""
|
||||||
"""
|
|
||||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||||
emit_previous_item_done_events,
|
emit_previous_item_done_events,
|
||||||
)
|
)
|
||||||
@@ -1185,8 +1184,7 @@ class TestHarmonyPreambleStreaming:
|
|||||||
|
|
||||||
def test_commentary_with_recipient_no_preamble_done(self) -> None:
|
def test_commentary_with_recipient_no_preamble_done(self) -> None:
|
||||||
"""Commentary + recipient='functions.X' should route to function call
|
"""Commentary + recipient='functions.X' should route to function call
|
||||||
done, not preamble done.
|
done, not preamble done."""
|
||||||
"""
|
|
||||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||||
emit_previous_item_done_events,
|
emit_previous_item_done_events,
|
||||||
)
|
)
|
||||||
@@ -1332,8 +1330,7 @@ def _mock_parser_with_reasoning(serving, delta_sequence: list[DeltaMessage]):
|
|||||||
class TestStreamingReasoningToContentTransition:
|
class TestStreamingReasoningToContentTransition:
|
||||||
"""Tests for _process_simple_streaming_events reasoning-to-content
|
"""Tests for _process_simple_streaming_events reasoning-to-content
|
||||||
transition, specifically the fix for mixed deltas that carry both
|
transition, specifically the fix for mixed deltas that carry both
|
||||||
reasoning and content simultaneously.
|
reasoning and content simultaneously."""
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_mixed_delta_reasoning_and_content_emits_reasoning_delta(
|
async def test_mixed_delta_reasoning_and_content_emits_reasoning_delta(
|
||||||
@@ -1343,8 +1340,7 @@ class TestStreamingReasoningToContentTransition:
|
|||||||
and content set (e.g. reasoning end and content start in the same
|
and content set (e.g. reasoning end and content start in the same
|
||||||
chunk), the trailing reasoning text must be emitted as a
|
chunk), the trailing reasoning text must be emitted as a
|
||||||
ResponseReasoningTextDeltaEvent and included in the
|
ResponseReasoningTextDeltaEvent and included in the
|
||||||
ResponseReasoningTextDoneEvent text.
|
ResponseReasoningTextDoneEvent text."""
|
||||||
"""
|
|
||||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||||
serving = _make_serving_instance_with_reasoning()
|
serving = _make_serving_instance_with_reasoning()
|
||||||
|
|
||||||
@@ -1413,8 +1409,7 @@ class TestStreamingReasoningToContentTransition:
|
|||||||
self, monkeypatch
|
self, monkeypatch
|
||||||
):
|
):
|
||||||
"""When the transition from reasoning to content is clean (no mixed
|
"""When the transition from reasoning to content is clean (no mixed
|
||||||
delta), no extra reasoning delta event should be emitted.
|
delta), no extra reasoning delta event should be emitted."""
|
||||||
"""
|
|
||||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||||
serving = _make_serving_instance_with_reasoning()
|
serving = _make_serving_instance_with_reasoning()
|
||||||
|
|
||||||
@@ -1475,8 +1470,7 @@ class TestStreamingReasoningToContentTransition:
|
|||||||
async def test_reasoning_only_stream_no_content(self, monkeypatch):
|
async def test_reasoning_only_stream_no_content(self, monkeypatch):
|
||||||
"""When the stream has only reasoning deltas and no content, the
|
"""When the stream has only reasoning deltas and no content, the
|
||||||
reasoning done event should be emitted at finalization with the
|
reasoning done event should be emitted at finalization with the
|
||||||
full accumulated text, and no text delta events should appear.
|
full accumulated text, and no text delta events should appear."""
|
||||||
"""
|
|
||||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||||
serving = _make_serving_instance_with_reasoning()
|
serving = _make_serving_instance_with_reasoning()
|
||||||
|
|
||||||
|
|||||||
@@ -143,8 +143,7 @@ async def test_streaming_output_consistency(client: OpenAI, model_name: str):
|
|||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_streaming_logprobs(client: OpenAI, model_name: str):
|
async def test_streaming_logprobs(client: OpenAI, model_name: str):
|
||||||
"""Test that streaming with logprobs returns valid logprob data on
|
"""Test that streaming with logprobs returns valid logprob data on
|
||||||
output_text.delta events and that top_logprobs has the requested count.
|
output_text.delta events and that top_logprobs has the requested count."""
|
||||||
"""
|
|
||||||
response = await client.responses.create(
|
response = await client.responses.create(
|
||||||
model=model_name,
|
model=model_name,
|
||||||
input="Say hello.",
|
input="Say hello.",
|
||||||
|
|||||||
@@ -124,8 +124,7 @@ class TestProcessorCompoundDeltas:
|
|||||||
|
|
||||||
def test_split_name_and_args_same_index(self):
|
def test_split_name_and_args_same_index(self):
|
||||||
"""Regression: parsers like KimiK2 emit name and args as separate
|
"""Regression: parsers like KimiK2 emit name and args as separate
|
||||||
DeltaToolCalls at the same index within one DeltaMessage.
|
DeltaToolCalls at the same index within one DeltaMessage."""
|
||||||
"""
|
|
||||||
tc_name = _make_tool_call(0, name="get_weather")
|
tc_name = _make_tool_call(0, name="get_weather")
|
||||||
tc_args = _make_tool_call(0, arguments='{"city":"SF"}')
|
tc_args = _make_tool_call(0, arguments='{"city":"SF"}')
|
||||||
delta = DeltaMessage(tool_calls=[tc_name, tc_args])
|
delta = DeltaMessage(tool_calls=[tc_name, tc_args])
|
||||||
@@ -141,8 +140,7 @@ class TestProcessorCompoundDeltas:
|
|||||||
|
|
||||||
def test_reasoning_to_content_transition(self):
|
def test_reasoning_to_content_transition(self):
|
||||||
"""Regression: the old special case in emit_delta handled this;
|
"""Regression: the old special case in emit_delta handled this;
|
||||||
now split_delta handles it generically.
|
now split_delta handles it generically."""
|
||||||
"""
|
|
||||||
processor = SimpleStreamingEventProcessor()
|
processor = SimpleStreamingEventProcessor()
|
||||||
_run_through_processor(processor, DeltaMessage(reasoning="think"))
|
_run_through_processor(processor, DeltaMessage(reasoning="think"))
|
||||||
assert processor.state.current_state == _StateType.REASONING
|
assert processor.state.current_state == _StateType.REASONING
|
||||||
|
|||||||
@@ -25,8 +25,7 @@ def _model_config() -> Mock:
|
|||||||
def test_completion_flag_forwarded_to_tok_params():
|
def test_completion_flag_forwarded_to_tok_params():
|
||||||
"""build_tok_params must forward return_token_offsets, defaulting to
|
"""build_tok_params must forward return_token_offsets, defaulting to
|
||||||
False (zero behavioral change for existing callers) and coercing JSON
|
False (zero behavioral change for existing callers) and coercing JSON
|
||||||
null to False via the bool() guard.
|
null to False via the bool() guard."""
|
||||||
"""
|
|
||||||
cfg = _model_config()
|
cfg = _model_config()
|
||||||
|
|
||||||
default = CompletionRequest(model="m", prompt="hi")
|
default = CompletionRequest(model="m", prompt="hi")
|
||||||
@@ -41,8 +40,7 @@ def test_completion_flag_forwarded_to_tok_params():
|
|||||||
|
|
||||||
def test_chat_flag_forwarded_to_tok_params():
|
def test_chat_flag_forwarded_to_tok_params():
|
||||||
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
|
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
|
||||||
its return_token_offsets forwarding is verified independently.
|
its return_token_offsets forwarding is verified independently."""
|
||||||
"""
|
|
||||||
cfg = _model_config()
|
cfg = _model_config()
|
||||||
messages = [{"role": "user", "content": "hi"}]
|
messages = [{"role": "user", "content": "hi"}]
|
||||||
|
|
||||||
@@ -66,8 +64,7 @@ def test_generate_request_token_offsets_default_none():
|
|||||||
|
|
||||||
def test_generate_request_token_offsets_survive_json_round_trip():
|
def test_generate_request_token_offsets_survive_json_round_trip():
|
||||||
"""GenerateRequest crosses the disagg serialization boundary; the
|
"""GenerateRequest crosses the disagg serialization boundary; the
|
||||||
tuple[int, int] offsets must survive model_dump and re-validate.
|
tuple[int, int] offsets must survive model_dump and re-validate."""
|
||||||
"""
|
|
||||||
req = GenerateRequest(
|
req = GenerateRequest(
|
||||||
token_ids=[10, 20],
|
token_ids=[10, 20],
|
||||||
sampling_params=SamplingParams(),
|
sampling_params=SamplingParams(),
|
||||||
|
|||||||
@@ -111,8 +111,7 @@ def test_basic_embed(server: RemoteOpenAIServer, model_name: str):
|
|||||||
|
|
||||||
def test_unsupported_input_type_rejected(server: RemoteOpenAIServer, model_name: str):
|
def test_unsupported_input_type_rejected(server: RemoteOpenAIServer, model_name: str):
|
||||||
"""An input_type not defined in the model's prompt config should be
|
"""An input_type not defined in the model's prompt config should be
|
||||||
rejected with a 400 error.
|
rejected with a 400 error."""
|
||||||
"""
|
|
||||||
body = {
|
body = {
|
||||||
"model": model_name,
|
"model": model_name,
|
||||||
"input_type": "nonexistent_type",
|
"input_type": "nonexistent_type",
|
||||||
@@ -139,8 +138,7 @@ def test_omitted_input_type_accepted(server: RemoteOpenAIServer, model_name: str
|
|||||||
|
|
||||||
def test_v1_v2_parity(server: RemoteOpenAIServer, model_name: str):
|
def test_v1_v2_parity(server: RemoteOpenAIServer, model_name: str):
|
||||||
"""v1 (OpenAI) and v2 (Cohere) endpoints should produce the same
|
"""v1 (OpenAI) and v2 (Cohere) endpoints should produce the same
|
||||||
float embeddings for a generic model.
|
float embeddings for a generic model."""
|
||||||
"""
|
|
||||||
texts = ["hello world"]
|
texts = ["hello world"]
|
||||||
v2 = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
|
v2 = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
|
||||||
v1 = _openai_embed(server, model_name, texts)
|
v1 = _openai_embed(server, model_name, texts)
|
||||||
|
|||||||
@@ -76,8 +76,7 @@ def test_single_text_parity(server: RemoteOpenAIServer):
|
|||||||
|
|
||||||
def test_batch_parity(server: RemoteOpenAIServer):
|
def test_batch_parity(server: RemoteOpenAIServer):
|
||||||
"""A batch of texts should produce equivalent embeddings via both APIs,
|
"""A batch of texts should produce equivalent embeddings via both APIs,
|
||||||
in the same order.
|
in the same order."""
|
||||||
"""
|
|
||||||
texts = [
|
texts = [
|
||||||
"machine learning",
|
"machine learning",
|
||||||
"deep learning",
|
"deep learning",
|
||||||
|
|||||||
@@ -241,8 +241,7 @@ class TestResolveTruncation:
|
|||||||
|
|
||||||
def test_truncate_none_with_max_tokens(self):
|
def test_truncate_none_with_max_tokens(self):
|
||||||
"""truncate=NONE should NOT set truncate_prompt_tokens; the
|
"""truncate=NONE should NOT set truncate_prompt_tokens; the
|
||||||
max_tokens limit is enforced separately via _check_max_tokens.
|
max_tokens limit is enforced separately via _check_max_tokens."""
|
||||||
"""
|
|
||||||
req = self._make_request(truncate="NONE", max_tokens=10)
|
req = self._make_request(truncate="NONE", max_tokens=10)
|
||||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||||
assert tokens is None
|
assert tokens is None
|
||||||
|
|||||||
@@ -240,8 +240,7 @@ async def test_long_text_embedding_1500_chars(
|
|||||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||||
):
|
):
|
||||||
"""Test embedding processing for ~1500 character long text
|
"""Test embedding processing for ~1500 character long text
|
||||||
(~1028 tokens, exceeding 512 token limit).
|
(~1028 tokens, exceeding 512 token limit)."""
|
||||||
"""
|
|
||||||
# Verify text length
|
# Verify text length
|
||||||
# Verify text has sufficient word count (approximately 1500 words)
|
# Verify text has sufficient word count (approximately 1500 words)
|
||||||
word_count = len(LONG_TEXT_1500_WORDS.split())
|
word_count = len(LONG_TEXT_1500_WORDS.split())
|
||||||
@@ -289,8 +288,7 @@ async def test_long_text_embedding_2500_chars(
|
|||||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||||
):
|
):
|
||||||
"""Test embedding processing for ~2500 character long text
|
"""Test embedding processing for ~2500 character long text
|
||||||
(~2048 tokens, requiring multiple chunks).
|
(~2048 tokens, requiring multiple chunks)."""
|
||||||
"""
|
|
||||||
# Verify text length
|
# Verify text length
|
||||||
# Verify text has sufficient word count (approximately 2500 words)
|
# Verify text has sufficient word count (approximately 2500 words)
|
||||||
word_count = len(LONG_TEXT_2500_WORDS.split())
|
word_count = len(LONG_TEXT_2500_WORDS.split())
|
||||||
@@ -382,8 +380,7 @@ async def test_chunked_vs_normal_consistency(
|
|||||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||||
):
|
):
|
||||||
"""Test consistency between chunked and
|
"""Test consistency between chunked and
|
||||||
normal processing (using short text).
|
normal processing (using short text)."""
|
||||||
"""
|
|
||||||
# Use a short text within the 512 token limit
|
# Use a short text within the 512 token limit
|
||||||
short_text = (
|
short_text = (
|
||||||
"Artificial intelligence technology is changing our world, "
|
"Artificial intelligence technology is changing our world, "
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Unit tests for Cohere embed protocol: build_typed_embeddings and its
|
"""Unit tests for Cohere embed protocol: build_typed_embeddings and its
|
||||||
underlying packing helpers, plus Cohere-specific serving helpers.
|
underlying packing helpers, plus Cohere-specific serving helpers."""
|
||||||
"""
|
|
||||||
|
|
||||||
import struct
|
import struct
|
||||||
|
|
||||||
|
|||||||
@@ -454,8 +454,7 @@ async def test_derender_completion_empty_generate_responses(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_derender_completion_logprobs(client):
|
async def test_derender_completion_logprobs(client):
|
||||||
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
|
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
|
||||||
flat-list structure is returned with non-empty tokens and text_offsets.
|
flat-list structure is returned with non-empty tokens and text_offsets."""
|
||||||
"""
|
|
||||||
gr1 = await _render_completion(client, "Hello world")
|
gr1 = await _render_completion(client, "Hello world")
|
||||||
ids1 = gr1["token_ids"][:3]
|
ids1 = gr1["token_ids"][:3]
|
||||||
token_id = ids1[0]
|
token_id = ids1[0]
|
||||||
|
|||||||
@@ -121,8 +121,7 @@ async def _disagg(
|
|||||||
|
|
||||||
def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
|
def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
|
||||||
"""[(name, json normalized args)] so key ordering / whitespace don't
|
"""[(name, json normalized args)] so key ordering / whitespace don't
|
||||||
cause false negatives.
|
cause false negatives."""
|
||||||
"""
|
|
||||||
return [
|
return [
|
||||||
(tc["function"]["name"], json.loads(tc["function"]["arguments"]))
|
(tc["function"]["name"], json.loads(tc["function"]["arguments"]))
|
||||||
for tc in (response_choice["message"].get("tool_calls") or [])
|
for tc in (response_choice["message"].get("tool_calls") or [])
|
||||||
@@ -144,8 +143,7 @@ async def _run_parity_case(
|
|||||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||||
) -> tuple[dict, dict]:
|
) -> tuple[dict, dict]:
|
||||||
"""Run the coupled request then feed its generated tokens into the
|
"""Run the coupled request then feed its generated tokens into the
|
||||||
disaggregated derender endpoint. Returns (coupled, disagg).
|
disaggregated derender endpoint. Returns (coupled, disagg)."""
|
||||||
"""
|
|
||||||
coupled = await _coupled(client, messages, **extra)
|
coupled = await _coupled(client, messages, **extra)
|
||||||
ch = coupled["choices"][0]
|
ch = coupled["choices"][0]
|
||||||
chat_request = {"model": MODEL, "messages": messages, **extra}
|
chat_request = {"model": MODEL, "messages": messages, **extra}
|
||||||
@@ -205,8 +203,7 @@ async def test_parity_tool_call(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_parity_reasoning_and_tool_call(client):
|
async def test_parity_reasoning_and_tool_call(client):
|
||||||
"""Combined reasoning + tool call parity means the highest drift risk
|
"""Combined reasoning + tool call parity means the highest drift risk
|
||||||
since it exercises both parser branches on the same output.
|
since it exercises both parser branches on the same output."""
|
||||||
"""
|
|
||||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||||
coupled, disagg = await _run_parity_case(
|
coupled, disagg = await _run_parity_case(
|
||||||
client,
|
client,
|
||||||
|
|||||||
@@ -455,8 +455,7 @@ async def test_chat_completion_render_with_sampling_params(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_completion_render_emits_token_offsets(client):
|
async def test_completion_render_emits_token_offsets(client):
|
||||||
"""With return_token_offsets, /v1/completions/render returns per-token
|
"""With return_token_offsets, /v1/completions/render returns per-token
|
||||||
(start, end) char offsets aligned with token_ids.
|
(start, end) char offsets aligned with token_ids."""
|
||||||
"""
|
|
||||||
prompt = "Hello, world."
|
prompt = "Hello, world."
|
||||||
response = await client.post(
|
response = await client.post(
|
||||||
"/v1/completions/render",
|
"/v1/completions/render",
|
||||||
@@ -481,8 +480,7 @@ async def test_completion_render_emits_token_offsets(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_completion_render_default_no_token_offsets(client):
|
async def test_completion_render_default_no_token_offsets(client):
|
||||||
"""Without the flag, token_offsets must be null (existing responses
|
"""Without the flag, token_offsets must be null (existing responses
|
||||||
unchanged).
|
unchanged)."""
|
||||||
"""
|
|
||||||
response = await client.post(
|
response = await client.post(
|
||||||
"/v1/completions/render",
|
"/v1/completions/render",
|
||||||
json={
|
json={
|
||||||
@@ -499,8 +497,7 @@ async def test_completion_render_default_no_token_offsets(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_chat_render_emits_token_offsets(client):
|
async def test_chat_render_emits_token_offsets(client):
|
||||||
"""With return_token_offsets, /v1/chat/completions/render returns
|
"""With return_token_offsets, /v1/chat/completions/render returns
|
||||||
per-token offsets relative to the templated prompt string.
|
per-token offsets relative to the templated prompt string."""
|
||||||
"""
|
|
||||||
response = await client.post(
|
response = await client.post(
|
||||||
"/v1/chat/completions/render",
|
"/v1/chat/completions/render",
|
||||||
json={
|
json={
|
||||||
@@ -674,8 +671,7 @@ async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags
|
|||||||
client,
|
client,
|
||||||
):
|
):
|
||||||
"""With a ``{% generation %}``-enabled template, the mask marks assistant
|
"""With a ``{% generation %}``-enabled template, the mask marks assistant
|
||||||
tokens and the masked tokens decode to the assistant content.
|
tokens and the masked tokens decode to the assistant content."""
|
||||||
"""
|
|
||||||
response = await client.post(
|
response = await client.post(
|
||||||
"/v1/chat/completions/render",
|
"/v1/chat/completions/render",
|
||||||
json={
|
json={
|
||||||
|
|||||||
@@ -25,8 +25,7 @@ def _base_payload() -> dict:
|
|||||||
|
|
||||||
def test_omitted_max_tokens_is_not_provided():
|
def test_omitted_max_tokens_is_not_provided():
|
||||||
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
||||||
server can apply its own default instead of the dataclass 16.
|
server can apply its own default instead of the dataclass 16."""
|
||||||
"""
|
|
||||||
req = GenerateRequest.model_validate(_base_payload())
|
req = GenerateRequest.model_validate(_base_payload())
|
||||||
# SamplingParams' dataclass default leaks through the parsed instance —
|
# SamplingParams' dataclass default leaks through the parsed instance —
|
||||||
# this is exactly the bug the server-side defaulting works around.
|
# this is exactly the bug the server-side defaulting works around.
|
||||||
@@ -36,8 +35,7 @@ def test_omitted_max_tokens_is_not_provided():
|
|||||||
|
|
||||||
def test_explicit_max_tokens_is_provided():
|
def test_explicit_max_tokens_is_provided():
|
||||||
"""Even when the client picks the same value as the dataclass default,
|
"""Even when the client picks the same value as the dataclass default,
|
||||||
it must register as explicitly set so the server won't override it.
|
it must register as explicitly set so the server won't override it."""
|
||||||
"""
|
|
||||||
payload = _base_payload()
|
payload = _base_payload()
|
||||||
payload["sampling_params"] = {"max_tokens": 16}
|
payload["sampling_params"] = {"max_tokens": 16}
|
||||||
req = GenerateRequest.model_validate(payload)
|
req = GenerateRequest.model_validate(payload)
|
||||||
@@ -69,8 +67,7 @@ def test_json_roundtrip_preserves_provided_keys():
|
|||||||
def test_internal_instance_construction_treats_all_as_provided():
|
def test_internal_instance_construction_treats_all_as_provided():
|
||||||
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
||||||
``SamplingParams`` instance, every field is considered explicitly set
|
``SamplingParams`` instance, every field is considered explicitly set
|
||||||
so server-side defaulting can't clobber values resolved upstream.
|
so server-side defaulting can't clobber values resolved upstream."""
|
||||||
"""
|
|
||||||
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
||||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
||||||
assert req.is_sampling_param_provided("max_tokens")
|
assert req.is_sampling_param_provided("max_tokens")
|
||||||
|
|||||||
@@ -22,8 +22,7 @@ def test_sanitize_message():
|
|||||||
|
|
||||||
class TestSanitizeMessageFilePaths:
|
class TestSanitizeMessageFilePaths:
|
||||||
"""sanitize_message should also strip file paths and traceback
|
"""sanitize_message should also strip file paths and traceback
|
||||||
frames, not just memory addresses - see #31683.
|
frames, not just memory addresses - see #31683."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_strips_traceback_style_frame(self):
|
def test_strips_traceback_style_frame(self):
|
||||||
msg = (
|
msg = (
|
||||||
@@ -45,8 +44,7 @@ class TestSanitizeMessageFilePaths:
|
|||||||
|
|
||||||
def test_strips_single_parent_container_path(self):
|
def test_strips_single_parent_container_path(self):
|
||||||
"""Regression: /app/server.py and /workspace/server.py (common in
|
"""Regression: /app/server.py and /workspace/server.py (common in
|
||||||
container deployments) were missed by the original {2,} quantifier.
|
container deployments) were missed by the original {2,} quantifier."""
|
||||||
"""
|
|
||||||
assert "/app/" not in sanitize_message("Error in /app/server.py")
|
assert "/app/" not in sanitize_message("Error in /app/server.py")
|
||||||
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
|
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
|
||||||
|
|
||||||
|
|||||||
@@ -24,8 +24,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
|||||||
|
|
||||||
def _fake_request(log_error_stack: bool = False) -> SimpleNamespace:
|
def _fake_request(log_error_stack: bool = False) -> SimpleNamespace:
|
||||||
"""Minimal stand-in for a FastAPI Request - just enough for the
|
"""Minimal stand-in for a FastAPI Request - just enough for the
|
||||||
handler to read req.app.state.args.log_error_stack.
|
handler to read req.app.state.args.log_error_stack."""
|
||||||
"""
|
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
app=SimpleNamespace(
|
app=SimpleNamespace(
|
||||||
state=SimpleNamespace(args=SimpleNamespace(log_error_stack=log_error_stack))
|
state=SimpleNamespace(args=SimpleNamespace(log_error_stack=log_error_stack))
|
||||||
@@ -36,8 +35,7 @@ def _fake_request(log_error_stack: bool = False) -> SimpleNamespace:
|
|||||||
|
|
||||||
class TestValidationErrorParamFallback:
|
class TestValidationErrorParamFallback:
|
||||||
"""Ensure `param` falls back to the Pydantic error's `loc` when no
|
"""Ensure `param` falls back to the Pydantic error's `loc` when no
|
||||||
custom VLLMValidationError context is present.
|
custom VLLMValidationError context is present."""
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("error_type", "msg"),
|
("error_type", "msg"),
|
||||||
@@ -61,8 +59,7 @@ class TestValidationErrorParamFallback:
|
|||||||
async def test_param_fallback_does_not_crash_on_non_dict_error(self):
|
async def test_param_fallback_does_not_crash_on_non_dict_error(self):
|
||||||
"""Schemathesis fuzzing found that errors[0] isn't always a dict.
|
"""Schemathesis fuzzing found that errors[0] isn't always a dict.
|
||||||
The fallback must not crash in that case - it should just leave
|
The fallback must not crash in that case - it should just leave
|
||||||
param as None instead of raising.
|
param as None instead of raising."""
|
||||||
"""
|
|
||||||
exc = RequestValidationError(["some unexpected non-dict error"])
|
exc = RequestValidationError(["some unexpected non-dict error"])
|
||||||
|
|
||||||
response = await validation_exception_handler(_fake_request(), exc)
|
response = await validation_exception_handler(_fake_request(), exc)
|
||||||
@@ -100,8 +97,7 @@ class TestCleanLocForParam:
|
|||||||
|
|
||||||
def test_all_internal_falls_back_to_raw_join(self):
|
def test_all_internal_falls_back_to_raw_join(self):
|
||||||
"""If every loc segment looks internal, fall back to the raw
|
"""If every loc segment looks internal, fall back to the raw
|
||||||
dot-join rather than returning an empty string.
|
dot-join rather than returning an empty string."""
|
||||||
"""
|
|
||||||
loc = ("function-wrap[__log_extra_fields__()]",)
|
loc = ("function-wrap[__log_extra_fields__()]",)
|
||||||
assert clean_loc_for_param(loc) == "function-wrap[__log_extra_fields__()]"
|
assert clean_loc_for_param(loc) == "function-wrap[__log_extra_fields__()]"
|
||||||
|
|
||||||
@@ -109,8 +105,7 @@ class TestCleanLocForParam:
|
|||||||
class TestValidationErrorDoesNotLeakServerPaths:
|
class TestValidationErrorDoesNotLeakServerPaths:
|
||||||
"""FastAPI sets endpoint_file/line/function/path on the exception
|
"""FastAPI sets endpoint_file/line/function/path on the exception
|
||||||
during routing, not in the constructor - so we set them manually
|
during routing, not in the constructor - so we set them manually
|
||||||
here to reproduce the real leak. See #31683.
|
here to reproduce the real leak. See #31683."""
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_handler_strips_endpoint_file_context(self):
|
async def test_handler_strips_endpoint_file_context(self):
|
||||||
|
|||||||
@@ -245,8 +245,7 @@ async def test_multiple_lora_adapters(
|
|||||||
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
||||||
):
|
):
|
||||||
"""Validate that many loras can be dynamically registered and inferenced
|
"""Validate that many loras can be dynamically registered and inferenced
|
||||||
with concurrently
|
with concurrently"""
|
||||||
"""
|
|
||||||
|
|
||||||
# This test file configures the server with --max-cpu-loras=2 and this test
|
# This test file configures the server with --max-cpu-loras=2 and this test
|
||||||
# will concurrently load 10 adapters, so it should flex the LRU cache
|
# will concurrently load 10 adapters, so it should flex the LRU cache
|
||||||
|
|||||||
@@ -72,8 +72,7 @@ class MockModelConfig:
|
|||||||
@pytest.fixture(params=get_args(SupportedTask))
|
@pytest.fixture(params=get_args(SupportedTask))
|
||||||
def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]:
|
def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]:
|
||||||
"""For each supported task, build an app with only that task's routers,
|
"""For each supported task, build an app with only that task's routers,
|
||||||
extract all routes, and return the task name and routes.
|
extract all routes, and return the task name and routes."""
|
||||||
"""
|
|
||||||
task = request.param
|
task = request.param
|
||||||
# Enable development mode to register all routes (including dev-only routes).
|
# Enable development mode to register all routes (including dev-only routes).
|
||||||
monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1")
|
monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1")
|
||||||
@@ -99,8 +98,7 @@ def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]
|
|||||||
|
|
||||||
def test_auto_discovered_protected_routes_require_auth(task_routes):
|
def test_auto_discovered_protected_routes_require_auth(task_routes):
|
||||||
"""For every auto-discovered route that starts with a guarded prefix,
|
"""For every auto-discovered route that starts with a guarded prefix,
|
||||||
verify that authentication is enforced.
|
verify that authentication is enforced."""
|
||||||
"""
|
|
||||||
task, routes = task_routes
|
task, routes = task_routes
|
||||||
app = _create_app_with_mock_routes(routes)
|
app = _create_app_with_mock_routes(routes)
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
@@ -134,8 +132,7 @@ def test_auto_discovered_protected_routes_require_auth(task_routes):
|
|||||||
|
|
||||||
def test_auto_discovered_unprotected_routes_no_auth(task_routes):
|
def test_auto_discovered_unprotected_routes_no_auth(task_routes):
|
||||||
"""For every auto-discovered route that does NOT start with a guarded
|
"""For every auto-discovered route that does NOT start with a guarded
|
||||||
prefix, verify that no authentication is required.
|
prefix, verify that no authentication is required."""
|
||||||
"""
|
|
||||||
task, routes = task_routes
|
task, routes = task_routes
|
||||||
app = _create_app_with_mock_routes(routes)
|
app = _create_app_with_mock_routes(routes)
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
|
|||||||
@@ -114,8 +114,7 @@ class TestHandlerOverrideIntegration:
|
|||||||
|
|
||||||
def test_customer_script_functions_auto_loaded(self, monkeypatch, tmp_path):
|
def test_customer_script_functions_auto_loaded(self, monkeypatch, tmp_path):
|
||||||
"""Test customer scenario: script functions automatically override
|
"""Test customer scenario: script functions automatically override
|
||||||
framework defaults.
|
framework defaults."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.sagemaker.config import (
|
from model_hosting_container_standards.sagemaker.config import (
|
||||||
SageMakerEnvVars,
|
SageMakerEnvVars,
|
||||||
@@ -176,8 +175,7 @@ async def custom_sagemaker_invocation_handler(request: Request):
|
|||||||
|
|
||||||
def test_customer_decorator_usage(self, monkeypatch, tmp_path):
|
def test_customer_decorator_usage(self, monkeypatch, tmp_path):
|
||||||
"""Test customer scenario: using @custom_ping_handler and
|
"""Test customer scenario: using @custom_ping_handler and
|
||||||
@custom_invocation_handler decorators.
|
@custom_invocation_handler decorators."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.sagemaker.config import (
|
from model_hosting_container_standards.sagemaker.config import (
|
||||||
SageMakerEnvVars,
|
SageMakerEnvVars,
|
||||||
@@ -233,8 +231,7 @@ async def my_invoke(request: Request):
|
|||||||
|
|
||||||
def test_handler_priority_order(self, monkeypatch, tmp_path):
|
def test_handler_priority_order(self, monkeypatch, tmp_path):
|
||||||
"""Test priority: @custom_ping_handler/@custom_invocation_handler
|
"""Test priority: @custom_ping_handler/@custom_invocation_handler
|
||||||
decorators vs script functions.
|
decorators vs script functions."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.sagemaker.config import (
|
from model_hosting_container_standards.sagemaker.config import (
|
||||||
SageMakerEnvVars,
|
SageMakerEnvVars,
|
||||||
@@ -307,8 +304,7 @@ async def custom_sagemaker_invocation_handler(request: Request):
|
|||||||
|
|
||||||
def test_environment_variable_script_loading(self, monkeypatch, tmp_path):
|
def test_environment_variable_script_loading(self, monkeypatch, tmp_path):
|
||||||
"""Test that environment variables correctly specify script location
|
"""Test that environment variables correctly specify script location
|
||||||
and loading.
|
and loading."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.sagemaker.config import (
|
from model_hosting_container_standards.sagemaker.config import (
|
||||||
SageMakerEnvVars,
|
SageMakerEnvVars,
|
||||||
@@ -420,8 +416,7 @@ async def custom_sagemaker_invocation_handler(request: Request):
|
|||||||
|
|
||||||
def test_handler_env_var_override(self, monkeypatch, tmp_path):
|
def test_handler_env_var_override(self, monkeypatch, tmp_path):
|
||||||
"""Test CUSTOM_FASTAPI_PING_HANDLER and CUSTOM_FASTAPI_INVOCATION_HANDLER
|
"""Test CUSTOM_FASTAPI_PING_HANDLER and CUSTOM_FASTAPI_INVOCATION_HANDLER
|
||||||
environment variable overrides.
|
environment variable overrides."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.common.fastapi.config import (
|
from model_hosting_container_standards.common.fastapi.config import (
|
||||||
FastAPIEnvVars,
|
FastAPIEnvVars,
|
||||||
@@ -515,8 +510,7 @@ async def custom_sagemaker_invocation_handler(request: Request):
|
|||||||
|
|
||||||
def test_env_var_priority_over_decorator_and_script(self, monkeypatch, tmp_path):
|
def test_env_var_priority_over_decorator_and_script(self, monkeypatch, tmp_path):
|
||||||
"""Test that environment variables have highest priority over decorators
|
"""Test that environment variables have highest priority over decorators
|
||||||
and script functions for both ping and invocation handlers.
|
and script functions for both ping and invocation handlers."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from model_hosting_container_standards.common.fastapi.config import (
|
from model_hosting_container_standards.common.fastapi.config import (
|
||||||
FastAPIEnvVars,
|
FastAPIEnvVars,
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ async def test_tokenize_then_chat_completion_with_image(
|
|||||||
local_asset_server,
|
local_asset_server,
|
||||||
):
|
):
|
||||||
"""Tokenize a multimodal message, then send the same message to chat
|
"""Tokenize a multimodal message, then send the same message to chat
|
||||||
completions. The chat completion must succeed (not 500).
|
completions. The chat completion must succeed (not 500)."""
|
||||||
"""
|
|
||||||
image_url = local_asset_server.url_for("stop_sign.jpg")
|
image_url = local_asset_server.url_for("stop_sign.jpg")
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -36,13 +36,11 @@ def test_should_include_usage_force_enables_continuous_usage(stream_options, exp
|
|||||||
class TestGetMaxTokens:
|
class TestGetMaxTokens:
|
||||||
"""Tests for get_max_tokens() to ensure generation_config's max_tokens
|
"""Tests for get_max_tokens() to ensure generation_config's max_tokens
|
||||||
acts as a default when from model author, and as a ceiling when
|
acts as a default when from model author, and as a ceiling when
|
||||||
explicitly set by the user.
|
explicitly set by the user."""
|
||||||
"""
|
|
||||||
|
|
||||||
def test_default_sampling_params_used_when_no_request_max_tokens(self):
|
def test_default_sampling_params_used_when_no_request_max_tokens(self):
|
||||||
"""When user doesn't specify max_tokens, generation_config default
|
"""When user doesn't specify max_tokens, generation_config default
|
||||||
should apply.
|
should apply."""
|
||||||
"""
|
|
||||||
result = get_max_tokens(
|
result = get_max_tokens(
|
||||||
max_model_len=24000,
|
max_model_len=24000,
|
||||||
max_tokens=None,
|
max_tokens=None,
|
||||||
@@ -53,8 +51,7 @@ class TestGetMaxTokens:
|
|||||||
|
|
||||||
def test_request_max_tokens_not_capped_by_default_sampling_params(self):
|
def test_request_max_tokens_not_capped_by_default_sampling_params(self):
|
||||||
"""When user specifies max_tokens in request, model author's
|
"""When user specifies max_tokens in request, model author's
|
||||||
generation_config max_tokens must NOT cap it (fixes #34005).
|
generation_config max_tokens must NOT cap it (fixes #34005)."""
|
||||||
"""
|
|
||||||
result = get_max_tokens(
|
result = get_max_tokens(
|
||||||
max_model_len=24000,
|
max_model_len=24000,
|
||||||
max_tokens=5000,
|
max_tokens=5000,
|
||||||
@@ -97,8 +94,7 @@ class TestGetMaxTokens:
|
|||||||
|
|
||||||
def test_request_max_tokens_smaller_than_default(self):
|
def test_request_max_tokens_smaller_than_default(self):
|
||||||
"""When user explicitly requests fewer tokens than gen_config default,
|
"""When user explicitly requests fewer tokens than gen_config default,
|
||||||
that should be respected.
|
that should be respected."""
|
||||||
"""
|
|
||||||
result = get_max_tokens(
|
result = get_max_tokens(
|
||||||
max_model_len=24000,
|
max_model_len=24000,
|
||||||
max_tokens=512,
|
max_tokens=512,
|
||||||
|
|||||||
@@ -239,8 +239,7 @@ def test_request_logger_log_outputs_integration():
|
|||||||
|
|
||||||
def test_streaming_complete_logs_full_text_content():
|
def test_streaming_complete_logs_full_text_content():
|
||||||
"""Test that streaming complete logging includes
|
"""Test that streaming complete logging includes
|
||||||
full accumulated text, not just token count.
|
full accumulated text, not just token count."""
|
||||||
"""
|
|
||||||
mock_logger = MagicMock()
|
mock_logger = MagicMock()
|
||||||
|
|
||||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||||
|
|||||||
@@ -17,8 +17,7 @@ from vllm.entrypoints.serve.utils.sse_keep_alive import (
|
|||||||
def test_disabled_returns_same_generator():
|
def test_disabled_returns_same_generator():
|
||||||
"""For 0, negative, and NaN intervals the exact same generator object must
|
"""For 0, negative, and NaN intervals the exact same generator object must
|
||||||
be returned (identity), proving no wrapper or per-chunk overhead when
|
be returned (identity), proving no wrapper or per-chunk overhead when
|
||||||
disabled.
|
disabled."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
yield "x"
|
yield "x"
|
||||||
@@ -34,8 +33,7 @@ def test_disabled_returns_same_generator():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_keep_alive_during_silent_start():
|
async def test_keep_alive_during_silent_start():
|
||||||
"""A silent start (queue wait / prefill) emits keep-alive comments before
|
"""A silent start (queue wait / prefill) emits keep-alive comments before
|
||||||
the first real chunk.
|
the first real chunk."""
|
||||||
"""
|
|
||||||
release = asyncio.Event()
|
release = asyncio.Event()
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
@@ -57,8 +55,7 @@ async def test_keep_alive_during_silent_start():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_no_keep_alive_when_streaming_is_fast():
|
async def test_no_keep_alive_when_streaming_is_fast():
|
||||||
"""If chunks arrive faster than the interval, no comment is emitted and all
|
"""If chunks arrive faster than the interval, no comment is emitted and all
|
||||||
data is forwarded.
|
data is forwarded."""
|
||||||
"""
|
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
for chunk in ["a", "b", "c"]:
|
for chunk in ["a", "b", "c"]:
|
||||||
@@ -72,8 +69,7 @@ async def test_no_keep_alive_when_streaming_is_fast():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_keep_alive_between_chunks_does_not_drop_data():
|
async def test_keep_alive_between_chunks_does_not_drop_data():
|
||||||
"""Keep-alives interleaved with real chunks must never drop or reorder
|
"""Keep-alives interleaved with real chunks must never drop or reorder
|
||||||
data.
|
data."""
|
||||||
"""
|
|
||||||
release_b = asyncio.Event()
|
release_b = asyncio.Event()
|
||||||
release_c = asyncio.Event()
|
release_c = asyncio.Event()
|
||||||
|
|
||||||
@@ -139,8 +135,7 @@ async def test_upstream_cancelled_error_propagates():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_aclose_closes_upstream():
|
async def test_aclose_closes_upstream():
|
||||||
"""Closing a wrapper that is mid-stream must complete promptly and close
|
"""Closing a wrapper that is mid-stream must complete promptly and close
|
||||||
the upstream generator (regression for the earlier leaked-task hang).
|
the upstream generator (regression for the earlier leaked-task hang)."""
|
||||||
"""
|
|
||||||
finalized = asyncio.Event()
|
finalized = asyncio.Event()
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
@@ -159,8 +154,7 @@ async def test_aclose_closes_upstream():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_client_disconnect_cancels_upstream():
|
async def test_client_disconnect_cancels_upstream():
|
||||||
"""Cancelling the consumer mid-stream must cancel the pending anext and
|
"""Cancelling the consumer mid-stream must cancel the pending anext and
|
||||||
close the upstream generator so its cleanup runs.
|
close the upstream generator so its cleanup runs."""
|
||||||
"""
|
|
||||||
finalized = asyncio.Event()
|
finalized = asyncio.Event()
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
@@ -187,8 +181,7 @@ async def test_client_disconnect_cancels_upstream():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_streaming_response_emits_comment():
|
async def test_streaming_response_emits_comment():
|
||||||
"""A keep-alive comment must reach the client through the real
|
"""A keep-alive comment must reach the client through the real
|
||||||
StreamingResponse used by the routers.
|
StreamingResponse used by the routers."""
|
||||||
"""
|
|
||||||
release = asyncio.Event()
|
release = asyncio.Event()
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
@@ -227,8 +220,7 @@ async def test_streaming_response_emits_comment():
|
|||||||
async def test_streaming_response_disconnect_closes_upstream():
|
async def test_streaming_response_disconnect_closes_upstream():
|
||||||
"""On the ASGI 2.3 disconnect path used by the pinned Uvicorn, Starlette
|
"""On the ASGI 2.3 disconnect path used by the pinned Uvicorn, Starlette
|
||||||
cancels streaming and the wrapper's finally must close the upstream
|
cancels streaming and the wrapper's finally must close the upstream
|
||||||
generator.
|
generator."""
|
||||||
"""
|
|
||||||
finalized = asyncio.Event()
|
finalized = asyncio.Event()
|
||||||
|
|
||||||
async def gen():
|
async def gen():
|
||||||
|
|||||||
@@ -295,8 +295,7 @@ async def test_session_update_invalid_model_returns_error(model_name):
|
|||||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
async def test_commit_without_session_update_returns_error(model_name):
|
async def test_commit_without_session_update_returns_error(model_name):
|
||||||
"""Test that committing before validating the model returns an error
|
"""Test that committing before validating the model returns an error
|
||||||
and does not fall through to processing.
|
and does not fall through to processing."""
|
||||||
"""
|
|
||||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||||
|
|
||||||
if model_name.startswith("mistralai"):
|
if model_name.startswith("mistralai"):
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ async def generate_mock_outputs(
|
|||||||
num_turns, prompt_token_counts, output_token_counts, cached_token_counts=None
|
num_turns, prompt_token_counts, output_token_counts, cached_token_counts=None
|
||||||
):
|
):
|
||||||
"""Generate a sequence of mock RequestOutput objects to simulate multiple
|
"""Generate a sequence of mock RequestOutput objects to simulate multiple
|
||||||
turns.
|
turns."""
|
||||||
"""
|
|
||||||
if cached_token_counts is None:
|
if cached_token_counts is None:
|
||||||
cached_token_counts = [0] * num_turns
|
cached_token_counts = [0] * num_turns
|
||||||
|
|
||||||
@@ -271,8 +270,7 @@ def test_reasoning_tokens_counting():
|
|||||||
|
|
||||||
def test_preamble_tokens_not_counted_as_reasoning():
|
def test_preamble_tokens_not_counted_as_reasoning():
|
||||||
"""Preambles (commentary with no recipient) are visible user text,
|
"""Preambles (commentary with no recipient) are visible user text,
|
||||||
not hidden reasoning. They must NOT inflate num_reasoning_tokens.
|
not hidden reasoning. They must NOT inflate num_reasoning_tokens."""
|
||||||
"""
|
|
||||||
context, parser = make_harmony_context()
|
context, parser = make_harmony_context()
|
||||||
parser.enqueue_chunk_result(reasoning_token_count=0)
|
parser.enqueue_chunk_result(reasoning_token_count=0)
|
||||||
|
|
||||||
@@ -289,8 +287,7 @@ def test_preamble_tokens_not_counted_as_reasoning():
|
|||||||
|
|
||||||
def test_commentary_with_recipient_counted_as_reasoning():
|
def test_commentary_with_recipient_counted_as_reasoning():
|
||||||
"""Commentary directed at a tool (recipient != None) is hidden from
|
"""Commentary directed at a tool (recipient != None) is hidden from
|
||||||
the user, so it should still count as reasoning tokens.
|
the user, so it should still count as reasoning tokens."""
|
||||||
"""
|
|
||||||
context, parser = make_harmony_context()
|
context, parser = make_harmony_context()
|
||||||
parser.enqueue_chunk_result(reasoning_token_count=3)
|
parser.enqueue_chunk_result(reasoning_token_count=3)
|
||||||
|
|
||||||
@@ -349,8 +346,7 @@ async def test_single_turn_no_tool_output():
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_negative_tool_tokens_edge_case():
|
async def test_negative_tool_tokens_edge_case():
|
||||||
"""Test edge case where calculation could result in negative tool
|
"""Test edge case where calculation could result in negative tool
|
||||||
tokens. We should log an error and clamp the value to 0.
|
tokens. We should log an error and clamp the value to 0."""
|
||||||
"""
|
|
||||||
# Use patch to check if logger.error was called
|
# Use patch to check if logger.error was called
|
||||||
with patch("vllm.entrypoints.openai.responses.context.logger.error") as mock_log:
|
with patch("vllm.entrypoints.openai.responses.context.logger.error") as mock_log:
|
||||||
context, _ = make_harmony_context(available_tools=["browser"])
|
context, _ = make_harmony_context(available_tools=["browser"])
|
||||||
@@ -690,8 +686,7 @@ def create_simple_context_output(
|
|||||||
finished=True,
|
finished=True,
|
||||||
):
|
):
|
||||||
"""Helper to create a RequestOutput with customizable text for
|
"""Helper to create a RequestOutput with customizable text for
|
||||||
SimpleContext tests.
|
SimpleContext tests."""
|
||||||
"""
|
|
||||||
if token_ids is None:
|
if token_ids is None:
|
||||||
token_ids = []
|
token_ids = []
|
||||||
return RequestOutput(
|
return RequestOutput(
|
||||||
@@ -875,8 +870,7 @@ def test_simple_context_final_output():
|
|||||||
|
|
||||||
def test_simple_context_output_messages_empty_text_with_tokens():
|
def test_simple_context_output_messages_empty_text_with_tokens():
|
||||||
"""output_messages should be returned when tokens exist even if text is
|
"""output_messages should be returned when tokens exist even if text is
|
||||||
empty (e.g. special tokens).
|
empty (e.g. special tokens)."""
|
||||||
"""
|
|
||||||
context = SimpleContext()
|
context = SimpleContext()
|
||||||
context.append_output(
|
context.append_output(
|
||||||
create_simple_context_output(
|
create_simple_context_output(
|
||||||
@@ -894,8 +888,7 @@ def test_simple_context_output_messages_empty_text_with_tokens():
|
|||||||
|
|
||||||
def test_simple_context_output_messages_no_mutation():
|
def test_simple_context_output_messages_no_mutation():
|
||||||
"""Each call to output_messages returns a fresh list; callers can't
|
"""Each call to output_messages returns a fresh list; callers can't
|
||||||
corrupt internal state.
|
corrupt internal state."""
|
||||||
"""
|
|
||||||
context = SimpleContext()
|
context = SimpleContext()
|
||||||
context.append_output(
|
context.append_output(
|
||||||
create_simple_context_output(
|
create_simple_context_output(
|
||||||
|
|||||||
@@ -121,8 +121,7 @@ def test_get_world_size_tp1(vllm_runner):
|
|||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
|
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
|
||||||
"""Test that init_weight_transfer_engine calls the engine's
|
"""Test that init_weight_transfer_engine calls the engine's
|
||||||
init_transfer_engine method.
|
init_transfer_engine method."""
|
||||||
"""
|
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU for this test")
|
pytest.skip("Need at least 1 GPU for this test")
|
||||||
|
|
||||||
|
|||||||
@@ -86,8 +86,7 @@ def _all_kernel_classes() -> list[type]:
|
|||||||
|
|
||||||
def _probe(cls: type):
|
def _probe(cls: type):
|
||||||
"""A bare kernel instance with a plausible config, so input_quant_key()
|
"""A bare kernel instance with a plausible config, so input_quant_key()
|
||||||
can be queried without the hardware-gated constructor.
|
can be queried without the hardware-gated constructor."""
|
||||||
"""
|
|
||||||
obj = cls.__new__(cls) # type: ignore[call-overload]
|
obj = cls.__new__(cls) # type: ignore[call-overload]
|
||||||
if issubclass(cls, NvFp4LinearKernel):
|
if issubclass(cls, NvFp4LinearKernel):
|
||||||
obj.config = NvFp4LinearLayerConfig()
|
obj.config = NvFp4LinearLayerConfig()
|
||||||
|
|||||||
@@ -247,8 +247,7 @@ def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
|
|||||||
|
|
||||||
class _FakeLinear:
|
class _FakeLinear:
|
||||||
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
||||||
.quant_method are read by get_and_maybe_dequant_weights().
|
.quant_method are read by get_and_maybe_dequant_weights()."""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, weight: torch.Tensor):
|
def __init__(self, weight: torch.Tensor):
|
||||||
self.weight = weight
|
self.weight = weight
|
||||||
@@ -319,8 +318,7 @@ class _FakeAttnMetadata:
|
|||||||
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||||
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
||||||
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
||||||
decode attention through W_UK/W_UV.
|
decode attention through W_UK/W_UV."""
|
||||||
"""
|
|
||||||
set_random_seed(1)
|
set_random_seed(1)
|
||||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||||
impl, kv_b_weight = _make_amx_mla_impl(
|
impl, kv_b_weight = _make_amx_mla_impl(
|
||||||
@@ -386,8 +384,7 @@ def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
|||||||
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
||||||
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
||||||
prefill batch with a mix of fresh and continued (cached-prefix)
|
prefill batch with a mix of fresh and continued (cached-prefix)
|
||||||
sequences.
|
sequences."""
|
||||||
"""
|
|
||||||
set_random_seed(2)
|
set_random_seed(2)
|
||||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||||
impl, kv_b_weight = _make_amx_mla_impl(
|
impl, kv_b_weight = _make_amx_mla_impl(
|
||||||
|
|||||||
@@ -616,8 +616,7 @@ def test_non_causal_autoselect_backend():
|
|||||||
)
|
)
|
||||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||||
that it cannot handle.
|
that it cannot handle."""
|
||||||
"""
|
|
||||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||||
|
|
||||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||||
@@ -628,8 +627,7 @@ def test_flash_attn_accepts_handled_fp8_variants(
|
|||||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||||
):
|
):
|
||||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.
|
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||||
"""
|
|
||||||
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
||||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||||
|
|
||||||
|
|||||||
@@ -225,8 +225,7 @@ def test_cutlass_mla_decode_cross_layer_view():
|
|||||||
"""The kernel must read the cache's page-dim stride instead of assuming
|
"""The kernel must read the cache's page-dim stride instead of assuming
|
||||||
pages are packed back-to-back. A per-layer view into a cross-layer
|
pages are packed back-to-back. A per-layer view into a cross-layer
|
||||||
(block-major) cache has stride(0) inflated by num_layers; outputs must
|
(block-major) cache has stride(0) inflated by num_layers; outputs must
|
||||||
match a contiguous cache holding the same data exactly.
|
match a contiguous cache holding the same data exactly."""
|
||||||
"""
|
|
||||||
device = torch.device("cuda:0")
|
device = torch.device("cuda:0")
|
||||||
torch.set_default_dtype(torch.bfloat16)
|
torch.set_default_dtype(torch.bfloat16)
|
||||||
torch.set_default_device(device)
|
torch.set_default_device(device)
|
||||||
|
|||||||
@@ -162,8 +162,7 @@ def test_fast_decode_plan_importable() -> None:
|
|||||||
@torch.inference_mode
|
@torch.inference_mode
|
||||||
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||||
"""On the first call fast_plan_decode must route through self.plan() and
|
"""On the first call fast_plan_decode must route through self.plan() and
|
||||||
flip vllm_first_call to False on the wrapper object.
|
flip vllm_first_call to False on the wrapper object."""
|
||||||
"""
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||||
|
|||||||
@@ -134,8 +134,7 @@ def make_quantized_kv_cache(
|
|||||||
head_size: int,
|
head_size: int,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
|
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
|
||||||
kv_scale, ref_kv_cache, is_nvfp4_kv).
|
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
|
||||||
"""
|
|
||||||
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
||||||
if is_nvfp4_kv:
|
if is_nvfp4_kv:
|
||||||
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
|
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
|
||||||
|
|||||||
@@ -73,8 +73,7 @@ def _strided_context_inputs(
|
|||||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||||
workspace (left in the fp8 cache layout for a plain fp8 cache).
|
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||||
"""
|
|
||||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||||
|
|||||||
@@ -89,8 +89,7 @@ all_case_info: list[tuple] = []
|
|||||||
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
||||||
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||||
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
|
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
|
||||||
scales must not surface as NaN in the merged output.
|
scales must not surface as NaN in the merged output."""
|
||||||
"""
|
|
||||||
num_tokens, num_heads, head_size = 6, 8, 128
|
num_tokens, num_heads, head_size = 6, 8, 128
|
||||||
prefix_output = torch.zeros(
|
prefix_output = torch.zeros(
|
||||||
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
||||||
|
|||||||
@@ -45,8 +45,7 @@ def kv_layout(request) -> KVCacheLayout:
|
|||||||
|
|
||||||
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
|
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
|
||||||
"""Per-layer physical stride order for the given layout; the 3-dim
|
"""Per-layer physical stride order for the given layout; the 3-dim
|
||||||
indexer side cache (H=1) is contiguous, so identity.
|
indexer side cache (H=1) is contiguous, so identity."""
|
||||||
"""
|
|
||||||
if ndim == 3:
|
if ndim == 3:
|
||||||
return (0, 1, 2)
|
return (0, 1, 2)
|
||||||
stride_order = layout.layer_view_order
|
stride_order = layout.layer_view_order
|
||||||
@@ -66,8 +65,7 @@ def _main_spec() -> FullAttentionSpec:
|
|||||||
|
|
||||||
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
|
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
|
||||||
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
|
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
|
||||||
derived the same way the production allocator does.
|
derived the same way the production allocator does."""
|
||||||
"""
|
|
||||||
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
|
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
|
||||||
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
|
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
|
||||||
|
|
||||||
@@ -77,8 +75,7 @@ def _allocate_main_kv_via_contract(
|
|||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Build the main KV cache exactly as the production allocator does for the
|
"""Build the main KV cache exactly as the production allocator does for the
|
||||||
given layout: allocate the physical (permuted) tensor, then expose the
|
given layout: allocate the physical (permuted) tensor, then expose the
|
||||||
inverse-permuted logical [B, H, N, C] view the backend sees.
|
inverse-permuted logical [B, H, N, C] view the backend sees."""
|
||||||
"""
|
|
||||||
logical_shape = _main_kv_logical_shape(num_pages)
|
logical_shape = _main_kv_logical_shape(num_pages)
|
||||||
stride_order = _layer_stride_order(layout, len(logical_shape))
|
stride_order = _layer_stride_order(layout, len(logical_shape))
|
||||||
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
||||||
@@ -1379,8 +1376,7 @@ def test_amd_decode_index_topk_end_to_end(
|
|||||||
@pytest.mark.parametrize("num_idx_heads", [1, 4])
|
@pytest.mark.parametrize("num_idx_heads", [1, 4])
|
||||||
def test_decode_index_topk_fp8(num_idx_heads: int):
|
def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||||
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
|
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
|
||||||
matches a reference computed from the dequantized FP8 values.
|
matches a reference computed from the dequantized FP8 values."""
|
||||||
"""
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
|
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
|
||||||
decode_query_len = 1
|
decode_query_len = 1
|
||||||
@@ -1653,8 +1649,7 @@ def test_prefill_sparse_attention_correctness(
|
|||||||
def test_main_cache_layout_contract():
|
def test_main_cache_layout_contract():
|
||||||
"""The standardized per-layer logical shape is [B, H, N, C] with packed
|
"""The standardized per-layer logical shape is [B, H, N, C] with packed
|
||||||
K/V content, and the legacy layout aliases resolve to the expected
|
K/V content, and the legacy layout aliases resolve to the expected
|
||||||
per-layer stride orders.
|
per-layer stride orders."""
|
||||||
"""
|
|
||||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||||
logical = _main_kv_logical_shape(nb)
|
logical = _main_kv_logical_shape(nb)
|
||||||
assert logical == (nb, h, bs, 2 * d)
|
assert logical == (nb, h, bs, 2 * d)
|
||||||
@@ -1778,8 +1773,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
|||||||
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
||||||
the allocator's logical view stays contiguous and squeezes (as
|
the allocator's logical view stays contiguous and squeezes (as
|
||||||
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
|
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
|
||||||
[num_blocks, block_size, head_dim] cache the kernels consume.
|
[num_blocks, block_size, head_dim] cache the kernels consume."""
|
||||||
"""
|
|
||||||
nb = 5
|
nb = 5
|
||||||
ispec = MLAAttentionSpec(
|
ispec = MLAAttentionSpec(
|
||||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||||
@@ -1799,8 +1793,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
|||||||
|
|
||||||
def test_hnd_allocation_is_packed_head_major():
|
def test_hnd_allocation_is_packed_head_major():
|
||||||
"""Under HND the backend-visible logical view is the packed head-major
|
"""Under HND the backend-visible logical view is the packed head-major
|
||||||
physical allocation.
|
physical allocation."""
|
||||||
"""
|
|
||||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||||
logical = _main_kv_logical_shape(nb)
|
logical = _main_kv_logical_shape(nb)
|
||||||
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
||||||
@@ -1828,8 +1821,7 @@ def test_hnd_allocation_is_packed_head_major():
|
|||||||
def test_main_cache_is_block_first_and_unpadded():
|
def test_main_cache_is_block_first_and_unpadded():
|
||||||
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
||||||
is used for the main GQA cache: its spec is unpadded and the physical
|
is used for the main GQA cache: its spec is unpadded and the physical
|
||||||
layout keeps num_blocks as the first dimension under both layouts.
|
layout keeps num_blocks as the first dimension under both layouts."""
|
||||||
"""
|
|
||||||
spec = _main_spec()
|
spec = _main_spec()
|
||||||
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
||||||
assert spec.page_size_padded is None
|
assert spec.page_size_padded is None
|
||||||
@@ -1851,8 +1843,7 @@ def _build_decode_inputs(
|
|||||||
):
|
):
|
||||||
"""Shared decode setup: uniform query tokens per request, a non-identity
|
"""Shared decode setup: uniform query tokens per request, a non-identity
|
||||||
block table, and topk indices selecting the current block plus older causal
|
block table, and topk indices selecting the current block plus older causal
|
||||||
blocks for each query token.
|
blocks for each query token."""
|
||||||
"""
|
|
||||||
active_batch = len(seq_lens_list)
|
active_batch = len(seq_lens_list)
|
||||||
batch = active_batch + num_padded_reqs
|
batch = active_batch + num_padded_reqs
|
||||||
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
||||||
@@ -2188,8 +2179,7 @@ def test_decode_sparse_attention_correctness(
|
|||||||
):
|
):
|
||||||
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
||||||
the decode-site cache feed, and the strided HND case fails if the kernel
|
the decode-site cache feed, and the strided HND case fails if the kernel
|
||||||
ignores the cache strides.
|
ignores the cache strides."""
|
||||||
"""
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
||||||
seq_lens_list, decode_query_len, num_padded_reqs
|
seq_lens_list, decode_query_len, num_padded_reqs
|
||||||
@@ -2239,8 +2229,7 @@ def test_decode_wrong_layout_breaks_parity():
|
|||||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||||
reorders the K/V content, so the decode output no longer matches the
|
reorders the K/V content, so the decode output no longer matches the
|
||||||
reference computed on the correct logical view. The mislabeled tensor keeps
|
reference computed on the correct logical view. The mislabeled tensor keeps
|
||||||
the same shape as the correct view, so the kernel stays in bounds.
|
the same shape as the correct view, so the kernel stays in bounds."""
|
||||||
"""
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
seq_lens_list = (130, 257)
|
seq_lens_list = (130, 257)
|
||||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
||||||
@@ -2275,8 +2264,7 @@ def test_decode_wrong_layout_breaks_parity():
|
|||||||
def test_main_cache_byte_identical_through_production_allocator():
|
def test_main_cache_byte_identical_through_production_allocator():
|
||||||
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
|
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
|
||||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||||
same shape, stride, and storage offset as the packed-HND allocation.
|
same shape, stride, and storage offset as the packed-HND allocation."""
|
||||||
"""
|
|
||||||
nb = 4
|
nb = 4
|
||||||
spec = _main_spec()
|
spec = _main_spec()
|
||||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||||
@@ -2291,8 +2279,7 @@ def test_main_cache_byte_identical_through_production_allocator():
|
|||||||
def test_padded_main_cache_is_flagged():
|
def test_padded_main_cache_is_flagged():
|
||||||
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
||||||
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
||||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided.
|
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
|
||||||
"""
|
|
||||||
|
|
||||||
def _require_unpadded_block_first(spec, stride_order):
|
def _require_unpadded_block_first(spec, stride_order):
|
||||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||||
@@ -2329,8 +2316,7 @@ def test_padded_main_cache_is_flagged():
|
|||||||
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
|
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
|
||||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||||
packed-content K/V split views. Writing through those views must persist
|
packed-content K/V split views. Writing through those views must persist
|
||||||
into the bound storage under both layouts.
|
into the bound storage under both layouts."""
|
||||||
"""
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
num_pages = 4
|
num_pages = 4
|
||||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||||
|
|||||||
@@ -23,8 +23,7 @@ pytestmark = pytest.mark.skipif(
|
|||||||
def test_concat_and_cache_mla_into_unified_slot_view():
|
def test_concat_and_cache_mla_into_unified_slot_view():
|
||||||
"""concat_and_cache_mla must write correctly into a per-layer view whose
|
"""concat_and_cache_mla must write correctly into a per-layer view whose
|
||||||
block stride is the full unified slot (block-major), with zero bleed into
|
block stride is the full unified slot (block-major), with zero bleed into
|
||||||
the other layers' segments of the same slot.
|
the other layers' segments of the same slot."""
|
||||||
"""
|
|
||||||
from vllm import _custom_ops as ops
|
from vllm import _custom_ops as ops
|
||||||
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
@@ -86,8 +85,7 @@ def test_concat_and_cache_mla_into_unified_slot_view():
|
|||||||
def test_flashmla_dense_decode_unified_slot_view():
|
def test_flashmla_dense_decode_unified_slot_view():
|
||||||
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
|
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
|
||||||
on Hopper) must read a unified-slot block-major view bit-identically to a
|
on Hopper) must read a unified-slot block-major view bit-identically to a
|
||||||
contiguous per-layer cache.
|
contiguous per-layer cache."""
|
||||||
"""
|
|
||||||
import vllm.v1.attention.ops.flashmla as fm
|
import vllm.v1.attention.ops.flashmla as fm
|
||||||
|
|
||||||
ok, reason = fm.is_flashmla_dense_supported()
|
ok, reason = fm.is_flashmla_dense_supported()
|
||||||
@@ -152,8 +150,7 @@ def test_flashmla_dense_decode_unified_slot_view():
|
|||||||
def test_flashinfer_mla_dense_decode_unified_slot_view():
|
def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||||
"""FlashInfer MLA dense decode must read a unified-slot block-major view
|
"""FlashInfer MLA dense decode must read a unified-slot block-major view
|
||||||
(inflated stride(0), non-zero storage offset) bit-identically to a
|
(inflated stride(0), non-zero storage offset) bit-identically to a
|
||||||
contiguous per-layer cache.
|
contiguous per-layer cache."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@@ -224,8 +221,7 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
|||||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||||
cache, with finite nonzero output.
|
cache, with finite nonzero output."""
|
||||||
"""
|
|
||||||
import vllm.v1.attention.ops.flashmla as fm
|
import vllm.v1.attention.ops.flashmla as fm
|
||||||
|
|
||||||
ok, reason = fm.is_flashmla_sparse_supported()
|
ok, reason = fm.is_flashmla_sparse_supported()
|
||||||
@@ -314,8 +310,7 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
|||||||
def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
||||||
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
|
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
|
||||||
write correctly into a per-layer view whose block stride is the full
|
write correctly into a per-layer view whose block stride is the full
|
||||||
unified slot, with zero bleed into the other layers' segments.
|
unified slot, with zero bleed into the other layers' segments."""
|
||||||
"""
|
|
||||||
from vllm import _custom_ops as ops
|
from vllm import _custom_ops as ops
|
||||||
|
|
||||||
torch.manual_seed(0)
|
torch.manual_seed(0)
|
||||||
@@ -360,8 +355,7 @@ def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
|||||||
|
|
||||||
def test_flashattn_mla_dense_decode_unified_slot_view():
|
def test_flashattn_mla_dense_decode_unified_slot_view():
|
||||||
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
|
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
|
||||||
block-major view bit-identically to a contiguous per-layer cache.
|
block-major view bit-identically to a contiguous per-layer cache."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@@ -434,8 +428,7 @@ def test_flashattn_mla_dense_decode_unified_slot_view():
|
|||||||
def test_flashmla_dense_fp8_decode_unified_slot_view():
|
def test_flashmla_dense_fp8_decode_unified_slot_view():
|
||||||
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
|
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
|
||||||
must read a unified-slot block-major view bit-identically to a contiguous
|
must read a unified-slot block-major view bit-identically to a contiguous
|
||||||
per-layer fp8 cache.
|
per-layer fp8 cache."""
|
||||||
"""
|
|
||||||
import vllm.v1.attention.ops.flashmla as fm
|
import vllm.v1.attention.ops.flashmla as fm
|
||||||
|
|
||||||
ok, reason = fm.is_flashmla_dense_supported()
|
ok, reason = fm.is_flashmla_dense_supported()
|
||||||
@@ -506,8 +499,7 @@ def test_flashmla_dense_fp8_decode_unified_slot_view():
|
|||||||
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
|
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
|
||||||
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
|
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
|
||||||
unified-slot block-major view bit-identically to a contiguous per-layer
|
unified-slot block-major view bit-identically to a contiguous per-layer
|
||||||
cache.
|
cache."""
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||||
except ImportError:
|
except ImportError:
|
||||||
|
|||||||
@@ -283,8 +283,7 @@ def _run_direct_flash_attn_case(
|
|||||||
# Backend contract tests --------------------------------------------------
|
# Backend contract tests --------------------------------------------------
|
||||||
def test_aiter_mha_backend_contract():
|
def test_aiter_mha_backend_contract():
|
||||||
"""The ROCm backend advertises the dtypes, shapes, and attention types it
|
"""The ROCm backend advertises the dtypes, shapes, and attention types it
|
||||||
is designed to handle.
|
is designed to handle."""
|
||||||
"""
|
|
||||||
from vllm.v1.attention.backend import AttentionType
|
from vllm.v1.attention.backend import AttentionType
|
||||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||||
|
|
||||||
@@ -312,8 +311,7 @@ def test_aiter_mha_backend_contract():
|
|||||||
|
|
||||||
def test_aiter_mha_backend_validates_kv_cache_block_size():
|
def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||||
"""The backend should reject KV cache specs that cannot be gathered
|
"""The backend should reject KV cache specs that cannot be gathered
|
||||||
correctly by the ROCm kernel.
|
correctly by the ROCm kernel."""
|
||||||
"""
|
|
||||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||||
|
|
||||||
@@ -333,8 +331,7 @@ def test_aiter_mha_backend_validates_kv_cache_block_size():
|
|||||||
|
|
||||||
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
|
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
|
||||||
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
|
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
|
||||||
capability tuple.
|
capability tuple."""
|
||||||
"""
|
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||||
|
|
||||||
@@ -359,8 +356,7 @@ def test_aiter_mha_env_flags_control_enablement(
|
|||||||
use_aiter, use_mha, expected, monkeypatch
|
use_aiter, use_mha, expected, monkeypatch
|
||||||
):
|
):
|
||||||
"""Both the global AITER flag and the MHA-specific flag must be enabled
|
"""Both the global AITER flag and the MHA-specific flag must be enabled
|
||||||
before the ROCm MHA path is considered active.
|
before the ROCm MHA path is considered active."""
|
||||||
"""
|
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
|
||||||
_assert_aiter_supported()
|
_assert_aiter_supported()
|
||||||
@@ -378,8 +374,7 @@ def test_aiter_mha_env_flags_control_enablement(
|
|||||||
|
|
||||||
def test_aiter_mha_platform_gate_matches_install_and_arch():
|
def test_aiter_mha_platform_gate_matches_install_and_arch():
|
||||||
"""The global AITER availability check should only open on ROCm MI3xx when
|
"""The global AITER availability check should only open on ROCm MI3xx when
|
||||||
the aiter package is installed.
|
the aiter package is installed."""
|
||||||
"""
|
|
||||||
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
|
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
|
||||||
|
|
||||||
assert is_aiter_found_and_supported() is (
|
assert is_aiter_found_and_supported() is (
|
||||||
@@ -650,8 +645,7 @@ def test_aiter_mha_multi_batch(num_heads, head_size, dtype):
|
|||||||
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
|
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
|
||||||
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
||||||
"""The direct paged-KV path should stay stable for both normal and very
|
"""The direct paged-KV path should stay stable for both normal and very
|
||||||
large block tables.
|
large block tables."""
|
||||||
"""
|
|
||||||
atol = 2e-2
|
atol = 2e-2
|
||||||
rtol = 2e-2
|
rtol = 2e-2
|
||||||
_assert_aiter_supported()
|
_assert_aiter_supported()
|
||||||
@@ -673,8 +667,7 @@ def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
|||||||
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
||||||
def test_aiter_fa_sliding_window_matches_reference():
|
def test_aiter_fa_sliding_window_matches_reference():
|
||||||
"""The direct kernel should respect the same sliding-window causal mask as
|
"""The direct kernel should respect the same sliding-window causal mask as
|
||||||
the naive reference implementation.
|
the naive reference implementation."""
|
||||||
"""
|
|
||||||
atol = 2e-2
|
atol = 2e-2
|
||||||
rtol = 2e-2
|
rtol = 2e-2
|
||||||
_assert_aiter_supported()
|
_assert_aiter_supported()
|
||||||
|
|||||||
@@ -111,8 +111,7 @@ def _make_mixed_common_metadata():
|
|||||||
|
|
||||||
def _patch_build_deps(monkeypatch, events=None):
|
def _patch_build_deps(monkeypatch, events=None):
|
||||||
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
|
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
|
||||||
on CPU.
|
on CPU."""
|
||||||
"""
|
|
||||||
|
|
||||||
def fake_generate_sparse_seqlen_triton(
|
def fake_generate_sparse_seqlen_triton(
|
||||||
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||||
@@ -153,8 +152,7 @@ def test_build_populates_decode_only_split_fields(monkeypatch):
|
|||||||
|
|
||||||
def test_build_populates_mixed_split_fields(monkeypatch):
|
def test_build_populates_mixed_split_fields(monkeypatch):
|
||||||
"""Mixed decode+prefill batch: split is reported, prefill fields stay
|
"""Mixed decode+prefill batch: split is reported, prefill fields stay
|
||||||
default because this impl always runs the MQA path.
|
default because this impl always runs the MQA path."""
|
||||||
"""
|
|
||||||
builder = _make_builder()
|
builder = _make_builder()
|
||||||
_patch_build_deps(monkeypatch)
|
_patch_build_deps(monkeypatch)
|
||||||
|
|
||||||
|
|||||||
@@ -125,8 +125,7 @@ def _quantize_to_fp8(tensor: torch.Tensor):
|
|||||||
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
|
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
|
||||||
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
|
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
|
||||||
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
|
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
|
||||||
and compare against BF16 reference output.
|
and compare against BF16 reference output."""
|
||||||
"""
|
|
||||||
assert CACHE_SIZE % PAGE_SIZE == 0
|
assert CACHE_SIZE % PAGE_SIZE == 0
|
||||||
dtype = torch.bfloat16
|
dtype = torch.bfloat16
|
||||||
seq_len = L
|
seq_len = L
|
||||||
@@ -249,8 +248,7 @@ def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SI
|
|||||||
"""The kernel must honor the cache's page-dim stride, not assume pages are
|
"""The kernel must honor the cache's page-dim stride, not assume pages are
|
||||||
packed back-to-back. A per-layer view into a cross-layer (block-major)
|
packed back-to-back. A per-layer view into a cross-layer (block-major)
|
||||||
cache has stride(0) inflated by num_layers; outputs must match a
|
cache has stride(0) inflated by num_layers; outputs must match a
|
||||||
contiguous cache holding the same data exactly.
|
contiguous cache holding the same data exactly."""
|
||||||
"""
|
|
||||||
B = 3
|
B = 3
|
||||||
seq_len = 1027
|
seq_len = 1027
|
||||||
CACHE_SIZE = 16384
|
CACHE_SIZE = 16384
|
||||||
|
|||||||
@@ -41,8 +41,7 @@ def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
|||||||
def test_fused_embed_norm_matches_reference():
|
def test_fused_embed_norm_matches_reference():
|
||||||
"""Main-model fusion: the residual is the exact gather and the second output
|
"""Main-model fusion: the residual is the exact gather and the second output
|
||||||
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
|
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
|
||||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel.
|
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
|
||||||
"""
|
|
||||||
set_random_seed(13)
|
set_random_seed(13)
|
||||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||||
@@ -61,8 +60,7 @@ def test_fused_embed_norm_matches_reference():
|
|||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def test_fused_embed_eh_norm_matches_reference():
|
def test_fused_embed_eh_norm_matches_reference():
|
||||||
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
|
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
|
||||||
feeding the untouched model-local ``fused_eh_norm``.
|
feeding the untouched model-local ``fused_eh_norm``."""
|
||||||
"""
|
|
||||||
set_random_seed(13)
|
set_random_seed(13)
|
||||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||||
|
|||||||
@@ -62,8 +62,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
|||||||
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
|
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
|
||||||
qr_out with row stride q_size + kv_size. Downstream dispatchers that
|
qr_out with row stride q_size + kv_size. Downstream dispatchers that
|
||||||
require packed row-major inputs then reject the tensor and silently fall
|
require packed row-major inputs then reject the tensor and silently fall
|
||||||
back to a slower GEMM path on every decode step.
|
back to a slower GEMM path on every decode step."""
|
||||||
"""
|
|
||||||
device = "cuda"
|
device = "cuda"
|
||||||
dtype = torch.bfloat16
|
dtype = torch.bfloat16
|
||||||
q_size, kv_size = 192, 576
|
q_size, kv_size = 192, 576
|
||||||
@@ -89,8 +88,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
|||||||
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
||||||
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
||||||
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
||||||
lifts that bound to 2**31-1.
|
lifts that bound to 2**31-1."""
|
||||||
"""
|
|
||||||
device = "cuda"
|
device = "cuda"
|
||||||
dtype = torch.bfloat16
|
dtype = torch.bfloat16
|
||||||
q_size, kv_size = 192, 576
|
q_size, kv_size = 192, 576
|
||||||
|
|||||||
@@ -2,8 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
|
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
|
||||||
matching the eager triton kernel output.
|
matching the eager triton kernel output."""
|
||||||
"""
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|||||||
@@ -579,8 +579,7 @@ def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
|
|||||||
|
|
||||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
||||||
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
||||||
contiguous inputs.
|
contiguous inputs."""
|
||||||
"""
|
|
||||||
require_aiter()
|
require_aiter()
|
||||||
require_fp8()
|
require_fp8()
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user