[V1][Metrics] Support Sliding Window Attention (SWA) and hybrid layers in MFU/MBU estimation (#55624)
Signed-off-by: Thillai Chithambaram <thillaichithambaram.a@gmail.com> Co-authored-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
co-authored by
Mark McLoughlin
parent
b7e8dd8f37
commit
435c96f9db
@@ -8,6 +8,7 @@ import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from transformers import Gemma2Config, MistralConfig
|
||||
from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config
|
||||
from transformers.models.llama4.configuration_llama4 import (
|
||||
Llama4Config,
|
||||
@@ -31,6 +32,7 @@ from vllm.v1.metrics.perf import (
|
||||
MLAAttentionMetrics,
|
||||
ModelMetrics,
|
||||
ParsedArgs,
|
||||
SlidingWindowAttentionParser,
|
||||
UnembedMetrics,
|
||||
)
|
||||
|
||||
@@ -38,7 +40,8 @@ from vllm.v1.metrics.perf import (
|
||||
class MockModelConfig:
|
||||
"""Mock ModelConfig that implements the getter methods used by parsers."""
|
||||
|
||||
def __init__(self, hf_config, dtype):
|
||||
def __init__(self, hf_config, dtype, disable_sliding_window=False):
|
||||
"""Initialize MockModelConfig with HF config and options."""
|
||||
self.hf_config = hf_config
|
||||
self.hf_text_config = get_hf_text_config(hf_config)
|
||||
convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get(
|
||||
@@ -49,8 +52,10 @@ class MockModelConfig:
|
||||
).convert()
|
||||
self.dtype = dtype
|
||||
self.is_attention_free = False
|
||||
self.disable_sliding_window = disable_sliding_window
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Delegate attribute lookups to ModelConfig."""
|
||||
# 1. Check if ModelConfig actually has this attribute
|
||||
if not hasattr(ModelConfig, name):
|
||||
raise AttributeError(
|
||||
@@ -86,9 +91,13 @@ def create_mock_vllm_config(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
enable_expert_parallel=False,
|
||||
disable_sliding_window=False,
|
||||
) -> SimpleNamespace:
|
||||
"""Create a mock VllmConfig object for unit tests."""
|
||||
vllm_config = SimpleNamespace()
|
||||
vllm_config.model_config = MockModelConfig(hf_config, model_dtype)
|
||||
vllm_config.model_config = MockModelConfig(
|
||||
hf_config, model_dtype, disable_sliding_window=disable_sliding_window
|
||||
)
|
||||
|
||||
vllm_config.cache_config = SimpleNamespace()
|
||||
vllm_config.cache_config.cache_dtype = cache_dtype
|
||||
@@ -1336,3 +1345,364 @@ def test_mla_attention_scaling_with_layers():
|
||||
assert double_metrics.get_num_flops(ctx) == 2 * base_metrics.get_num_flops(ctx)
|
||||
assert double_metrics.get_read_bytes(ctx) == 2 * base_metrics.get_read_bytes(ctx)
|
||||
assert double_metrics.get_write_bytes(ctx) == 2 * base_metrics.get_write_bytes(ctx)
|
||||
|
||||
|
||||
#### Sliding Window Attention Tests ####
|
||||
|
||||
|
||||
def test_sliding_window_parser_no_swa():
|
||||
"""Test parser defaults to no SWA when sliding_window is unset."""
|
||||
hf_config = Qwen3Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_hidden_layers=12,
|
||||
)
|
||||
vllm_config = create_mock_vllm_config(hf_config)
|
||||
parser = SlidingWindowAttentionParser()
|
||||
args = ParsedArgs()
|
||||
args.num_hidden_layers = 12
|
||||
result = parser.parse(args, vllm_config)
|
||||
|
||||
assert result.sliding_window is None
|
||||
assert result.num_swa_layers == 0
|
||||
|
||||
|
||||
def test_sliding_window_parser_uniform_swa():
|
||||
"""Test SlidingWindowAttentionParser for full SWA model (e.g. Mistral)."""
|
||||
hf_config = MistralConfig(
|
||||
hidden_size=4096,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=8,
|
||||
num_hidden_layers=32,
|
||||
sliding_window=4096,
|
||||
)
|
||||
vllm_config = create_mock_vllm_config(hf_config)
|
||||
parser = SlidingWindowAttentionParser()
|
||||
args = ParsedArgs()
|
||||
args.num_hidden_layers = 32
|
||||
result = parser.parse(args, vllm_config)
|
||||
|
||||
assert result.sliding_window == 4096
|
||||
assert result.num_swa_layers == 32
|
||||
|
||||
|
||||
def test_sliding_window_parser_explicit_layer_types():
|
||||
"""Test SlidingWindowAttentionParser with explicit layer_types list."""
|
||||
hf_config = Qwen3Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_hidden_layers=6,
|
||||
)
|
||||
hf_config.sliding_window = 2048
|
||||
hf_config.layer_types = [
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
]
|
||||
vllm_config = create_mock_vllm_config(hf_config)
|
||||
parser = SlidingWindowAttentionParser()
|
||||
args = ParsedArgs()
|
||||
args.num_hidden_layers = 6
|
||||
result = parser.parse(args, vllm_config)
|
||||
|
||||
assert result.sliding_window == 2048
|
||||
assert result.num_swa_layers == 3
|
||||
|
||||
|
||||
def test_sliding_window_parser_gemma2():
|
||||
"""Test SlidingWindowAttentionParser detects alternating SWA for Gemma 2."""
|
||||
hf_config = Gemma2Config(
|
||||
hidden_size=2304,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=4,
|
||||
num_hidden_layers=26,
|
||||
sliding_window=4096,
|
||||
)
|
||||
vllm_config = create_mock_vllm_config(hf_config)
|
||||
parser = SlidingWindowAttentionParser()
|
||||
args = ParsedArgs()
|
||||
args.num_hidden_layers = 26
|
||||
result = parser.parse(args, vllm_config)
|
||||
|
||||
assert result.sliding_window == 4096
|
||||
# Gemma 2 alternates: even layers are sliding window (26 layers -> 13 SWA)
|
||||
assert result.num_swa_layers == 13
|
||||
|
||||
|
||||
def test_sliding_window_parser_disabled():
|
||||
"""Test SlidingWindowAttentionParser respects disable_sliding_window."""
|
||||
hf_config = MistralConfig(
|
||||
hidden_size=4096,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=8,
|
||||
num_hidden_layers=32,
|
||||
sliding_window=4096,
|
||||
)
|
||||
vllm_config = create_mock_vllm_config(hf_config, disable_sliding_window=True)
|
||||
parser = SlidingWindowAttentionParser()
|
||||
args = ParsedArgs()
|
||||
args.num_hidden_layers = 32
|
||||
result = parser.parse(args, vllm_config)
|
||||
|
||||
assert result.sliding_window is None
|
||||
assert result.num_swa_layers == 0
|
||||
|
||||
|
||||
def test_attention_metrics_sliding_window_flops_bounds():
|
||||
"""Test that SWA clamps attention core FLOPs when context_len > sliding_window."""
|
||||
sw = 1024
|
||||
dense_config = MistralConfig(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=8,
|
||||
sliding_window=None,
|
||||
)
|
||||
swa_config = MistralConfig(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=8,
|
||||
sliding_window=sw,
|
||||
)
|
||||
|
||||
dense_vllm = create_mock_vllm_config(dense_config)
|
||||
swa_vllm = create_mock_vllm_config(swa_config)
|
||||
|
||||
dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm)
|
||||
swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm)
|
||||
|
||||
# 1. When context <= sliding_window: FLOPs should be identical
|
||||
short_ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=512, is_prefill=False
|
||||
)
|
||||
assert swa_metrics.get_num_flops(short_ctx) == dense_metrics.get_num_flops(
|
||||
short_ctx
|
||||
)
|
||||
|
||||
# 2. When context > sliding_window: FLOPs for SWA must be strictly smaller
|
||||
long_ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=8192, is_prefill=False
|
||||
)
|
||||
dense_flops = dense_metrics.get_num_flops_breakdown(long_ctx)
|
||||
swa_flops = swa_metrics.get_num_flops_breakdown(long_ctx)
|
||||
|
||||
# Projections (qkv_proj, out_proj) are unaffected by sliding window
|
||||
assert swa_flops["qkv_proj"] == dense_flops["qkv_proj"]
|
||||
assert swa_flops["out_proj"] == dense_flops["out_proj"]
|
||||
|
||||
# Attention core FLOPs are clamped: 8192 vs 1024 (8x difference)
|
||||
assert swa_flops["attn_qk"] < dense_flops["attn_qk"]
|
||||
assert swa_flops["attn_av"] < dense_flops["attn_av"]
|
||||
assert swa_flops["attn_qk"] * 8 == dense_flops["attn_qk"]
|
||||
assert swa_flops["attn_av"] * 8 == dense_flops["attn_av"]
|
||||
|
||||
|
||||
def test_attention_metrics_sliding_window_read_bytes():
|
||||
"""Test that SWA bounds decode KV-cache read bytes to sliding_window."""
|
||||
sw = 2048
|
||||
dense_config = MistralConfig(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=4,
|
||||
num_hidden_layers=4,
|
||||
sliding_window=None,
|
||||
)
|
||||
swa_config = MistralConfig(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=4,
|
||||
num_hidden_layers=4,
|
||||
sliding_window=sw,
|
||||
)
|
||||
|
||||
dense_vllm = create_mock_vllm_config(dense_config)
|
||||
swa_vllm = create_mock_vllm_config(swa_config)
|
||||
|
||||
dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm)
|
||||
swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm)
|
||||
|
||||
# Long decode context: 8192 tokens
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=8192, is_prefill=False
|
||||
)
|
||||
dense_reads = dense_metrics.get_read_bytes_breakdown(ctx)
|
||||
swa_reads = swa_metrics.get_read_bytes_breakdown(ctx)
|
||||
|
||||
# Attention input read traffic is lower for SWA
|
||||
assert swa_reads["attn_input"] < dense_reads["attn_input"]
|
||||
|
||||
# Weight reads and projection inputs are identical
|
||||
assert swa_reads["qkv_weight"] == dense_reads["qkv_weight"]
|
||||
assert swa_reads["out_weight"] == dense_reads["out_weight"]
|
||||
assert swa_reads["qkv_input"] == dense_reads["qkv_input"]
|
||||
|
||||
|
||||
def test_attention_metrics_hybrid_alternating_layers():
|
||||
"""Test hybrid model where half layers are full attention and half are SWA."""
|
||||
sw = 1024
|
||||
hybrid_config = Qwen3Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=4,
|
||||
)
|
||||
hybrid_config.sliding_window = sw
|
||||
hybrid_config.layer_types = [
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
]
|
||||
|
||||
vllm_config = create_mock_vllm_config(hybrid_config)
|
||||
metrics = AttentionMetrics.from_vllm_config(vllm_config)
|
||||
|
||||
assert metrics.num_swa_layers == 2
|
||||
assert metrics.sliding_window == sw
|
||||
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=4096, is_prefill=False
|
||||
)
|
||||
flops = metrics.get_num_flops_breakdown(ctx)
|
||||
|
||||
# 2 full layers attend to 4096; 2 SWA layers attend to 1024
|
||||
# Expected TC total = 2 * 4096 + 2 * 1024 = 8192 + 2048 = 10240
|
||||
# Formula: 2 * q * d * (L_full * 4096 + L_swa * 1024)
|
||||
q = metrics.num_attention_heads
|
||||
d = metrics.head_dim
|
||||
expected_core_flops = 2 * q * d * (2 * 4096 + 2 * 1024)
|
||||
assert flops["attn_qk"] == expected_core_flops
|
||||
assert flops["attn_av"] == expected_core_flops
|
||||
|
||||
|
||||
def test_attention_metrics_sliding_window_per_gpu():
|
||||
"""Test that SWA metrics scale properly with tensor and pipeline parallelism."""
|
||||
sw = 1024
|
||||
hf_config = MistralConfig(
|
||||
hidden_size=4096,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=8,
|
||||
num_hidden_layers=16,
|
||||
sliding_window=sw,
|
||||
)
|
||||
single_vllm = create_mock_vllm_config(hf_config)
|
||||
tp_vllm = create_mock_vllm_config(hf_config, tensor_parallel_size=2)
|
||||
pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2)
|
||||
|
||||
single_metrics = AttentionMetrics.from_vllm_config(single_vllm)
|
||||
tp_metrics = AttentionMetrics.from_vllm_config(tp_vllm)
|
||||
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
|
||||
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=4096, is_prefill=False
|
||||
)
|
||||
|
||||
# Both TP=2 and PP=2 should halve the per-GPU attention core FLOPs
|
||||
single_flops = single_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
tp_flops = tp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
|
||||
assert tp_flops["attn_qk"] == single_flops["attn_qk"] // 2
|
||||
assert pp_flops["attn_qk"] == single_flops["attn_qk"] // 2
|
||||
|
||||
|
||||
def test_attention_metrics_hybrid_sliding_window_per_gpu():
|
||||
"""Test that hybrid SWA models preserve L_full + L_swa == L under PP."""
|
||||
sw = 1024
|
||||
hybrid_config = Gemma2Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=30,
|
||||
sliding_window=sw,
|
||||
)
|
||||
pp_vllm = create_mock_vllm_config(hybrid_config, pipeline_parallel_size=2)
|
||||
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
|
||||
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=4096, is_prefill=False
|
||||
)
|
||||
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
pp_read_bytes = pp_metrics.get_read_bytes_breakdown(ctx, per_gpu=True)
|
||||
|
||||
# Under PP=2, each global count is halved: 30 layers -> 15 per GPU, and the
|
||||
# 15 SWA / 15 full split becomes 7.5 each. Gemma 2 alternates, so an even
|
||||
# split is the correct average stage; deriving it after reducing L instead
|
||||
# produced a lopsided 8/7.
|
||||
q = pp_metrics.num_attention_heads
|
||||
d = pp_metrics.head_dim
|
||||
expected_pp_attn = int(2 * q * d * (7.5 * 4096 + 7.5 * sw))
|
||||
assert pp_flops["attn_qk"] == expected_pp_attn
|
||||
assert pp_flops["attn_av"] == expected_pp_attn
|
||||
assert "attn_input" in pp_read_bytes
|
||||
|
||||
|
||||
def test_attention_metrics_minority_swa_survives_pipeline_parallelism():
|
||||
"""A minority SWA sublayer must not round away under PP.
|
||||
|
||||
Regression test for a 6-layer model with a single sliding-window layer at
|
||||
pp_size=2: deriving the split after reducing L gave
|
||||
round(3 * 1 / 6) == 0, billing all three per-GPU layers as full attention
|
||||
and dropping the sliding-window contribution entirely.
|
||||
"""
|
||||
sw = 512
|
||||
context_len = 4096
|
||||
hf_config = Qwen3Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=6,
|
||||
)
|
||||
hf_config.sliding_window = sw
|
||||
hf_config.layer_types = ["sliding_attention"] + ["full_attention"] * 5
|
||||
|
||||
pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2)
|
||||
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
|
||||
assert pp_metrics.num_swa_layers == 1
|
||||
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=1, context_len=context_len, is_prefill=False
|
||||
)
|
||||
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
|
||||
# 5 full and 1 SWA layer, each halved by PP=2: 2.5 full and 0.5 SWA.
|
||||
q = pp_metrics.num_attention_heads
|
||||
d = pp_metrics.head_dim
|
||||
expected = int(2 * q * d * (2.5 * context_len + 0.5 * sw))
|
||||
assert pp_flops["attn_qk"] == expected
|
||||
assert pp_flops["attn_av"] == expected
|
||||
|
||||
# The sliding-window layer must change the result, i.e. it is not being
|
||||
# billed at the full context length.
|
||||
all_full = int(2 * q * d * 3 * context_len)
|
||||
assert pp_flops["attn_qk"] < all_full
|
||||
|
||||
|
||||
def test_attention_metrics_per_gpu_layers_sum_to_whole_model():
|
||||
"""Per-GPU counts times pp_size must account for every layer.
|
||||
|
||||
Flooring the layer count per stage silently drops the remainder, so a model
|
||||
whose layers do not divide evenly across stages under-reports its FLOPs.
|
||||
"""
|
||||
hf_config = Qwen3Config(
|
||||
hidden_size=2048,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
num_hidden_layers=7,
|
||||
)
|
||||
pp_size = 2
|
||||
vllm_config = create_mock_vllm_config(hf_config, pipeline_parallel_size=pp_size)
|
||||
metrics = AttentionMetrics.from_vllm_config(vllm_config)
|
||||
|
||||
ctx = ExecutionContext.from_single_request(
|
||||
num_tokens=8, context_len=1024, is_prefill=True
|
||||
)
|
||||
per_gpu = metrics.get_num_flops_breakdown(ctx, per_gpu=True)
|
||||
whole = metrics.get_num_flops_breakdown(ctx, per_gpu=False)
|
||||
|
||||
for key, value in whole.items():
|
||||
assert per_gpu[key] * pp_size == pytest.approx(value, rel=1e-9)
|
||||
|
||||
+268
-46
@@ -10,7 +10,7 @@ import json
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
import prometheus_client
|
||||
@@ -124,8 +124,12 @@ class ExecutionContext:
|
||||
decode_context_len: int = 0 # sum of context_len for decode requests
|
||||
decode_token_context_product: int = 0 # sum of (num_tokens * context_len)
|
||||
|
||||
# Per-request records: (num_tokens, context_len, is_prefill)
|
||||
requests: list[tuple[int, int, bool]] = field(default_factory=list)
|
||||
|
||||
def add(self, num_tokens: int, context_len: int, is_prefill: bool) -> None:
|
||||
"""Add a single request's statistics to this batch context."""
|
||||
self.requests.append((num_tokens, context_len, is_prefill))
|
||||
if is_prefill:
|
||||
self.num_prefill_requests += 1
|
||||
self.prefill_num_tokens += num_tokens
|
||||
@@ -141,9 +145,40 @@ class ExecutionContext:
|
||||
"""Total number of tokens across all requests in the batch."""
|
||||
return self.prefill_num_tokens + self.decode_num_tokens
|
||||
|
||||
def total_token_context_product(self) -> int:
|
||||
"""Total sum of (num_tokens * context_len) across all requests."""
|
||||
return self.prefill_token_context_product + self.decode_token_context_product
|
||||
def total_token_context_product(self, sliding_window: int | None = None) -> int:
|
||||
"""Total sum of (num_tokens * context_len) across all requests.
|
||||
|
||||
If sliding_window is provided, context length per request is clamped to
|
||||
sliding_window.
|
||||
"""
|
||||
if sliding_window is None:
|
||||
return (
|
||||
self.prefill_token_context_product + self.decode_token_context_product
|
||||
)
|
||||
return sum(
|
||||
num_tokens * min(context_len, sliding_window)
|
||||
for num_tokens, context_len, _ in self.requests
|
||||
)
|
||||
|
||||
def prefill_context_len_for_window(self, sliding_window: int | None = None) -> int:
|
||||
"""Sum of context_len for prefill requests, clamped to sliding_window if set."""
|
||||
if sliding_window is None:
|
||||
return self.prefill_context_len
|
||||
return sum(
|
||||
min(context_len, sliding_window)
|
||||
for _, context_len, is_prefill in self.requests
|
||||
if is_prefill
|
||||
)
|
||||
|
||||
def decode_context_len_for_window(self, sliding_window: int | None = None) -> int:
|
||||
"""Sum of context_len for decode requests, clamped to sliding_window if set."""
|
||||
if sliding_window is None:
|
||||
return self.decode_context_len
|
||||
return sum(
|
||||
min(context_len, sliding_window)
|
||||
for _, context_len, is_prefill in self.requests
|
||||
if not is_prefill
|
||||
)
|
||||
|
||||
def num_logits_tokens(self) -> int:
|
||||
"""Number of tokens that require logits computation (unembedding).
|
||||
@@ -214,6 +249,7 @@ class ParserChain:
|
||||
self.parsers.append(parser)
|
||||
|
||||
def parse(self, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Apply all parsers in sequence and return the accumulated ParsedArgs."""
|
||||
args = ParsedArgs()
|
||||
for parser in self.parsers:
|
||||
args = parser.parse(args, vllm_config)
|
||||
@@ -234,7 +270,9 @@ class ComponentMetrics(BaseModel, ABC):
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def component_type(cls) -> str: ...
|
||||
def component_type(cls) -> str:
|
||||
"""Return the unique string key identifying this component in the registry."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
@@ -273,17 +311,23 @@ class ComponentMetrics(BaseModel, ABC):
|
||||
@abstractmethod
|
||||
def get_num_flops_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]: ...
|
||||
) -> dict[str, int]:
|
||||
"""Return per-operation FLOPs breakdown for this component."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_read_bytes_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]: ...
|
||||
) -> dict[str, int]:
|
||||
"""Return per-operation read-memory-traffic breakdown for this component."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_write_bytes_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]: ...
|
||||
) -> dict[str, int]:
|
||||
"""Return per-operation write-memory-traffic breakdown for this component."""
|
||||
...
|
||||
|
||||
def get_num_flops(self, ctx: ExecutionContext, per_gpu: bool = True) -> int:
|
||||
return sum(self.get_num_flops_breakdown(ctx, per_gpu).values())
|
||||
@@ -306,6 +350,7 @@ class BaseConfigParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Parse base model dimensions, dtype byte sizes, and parallelism config."""
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
args.vocab_size = model_config.get_vocab_size()
|
||||
@@ -357,6 +402,7 @@ class BaseAttentionConfigParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Parse KV head count, head dimension, and KV-cache dtype byte size."""
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
args.num_key_value_heads = model_config.get_total_num_kv_heads()
|
||||
@@ -371,6 +417,54 @@ class BaseAttentionConfigParser(Parser):
|
||||
return args
|
||||
|
||||
|
||||
class SlidingWindowAttentionParser(Parser):
|
||||
"""
|
||||
Parses sliding window attention configuration and layer breakdown.
|
||||
Provides: sliding_window, num_swa_layers
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Determine sliding_window size and count of SWA layers.
|
||||
|
||||
Detects per-layer type lists (e.g. Llama-4), Gemma 2 alternating
|
||||
pattern, and uniform SWA models (Mistral, Qwen2.5).
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
sliding_window: int | None = None
|
||||
if not getattr(model_config, "disable_sliding_window", False):
|
||||
sliding_window = model_config.get_sliding_window()
|
||||
if sliding_window is None and hasattr(vllm_config, "cache_config"):
|
||||
sliding_window = getattr(
|
||||
vllm_config.cache_config, "sliding_window", None
|
||||
)
|
||||
|
||||
args.sliding_window = sliding_window
|
||||
|
||||
num_layers = args.num_hidden_layers
|
||||
if sliding_window is not None and sliding_window > 0:
|
||||
hf_text_config = getattr(model_config, "hf_text_config", None)
|
||||
layer_types = getattr(hf_text_config, "layer_types", None)
|
||||
if layer_types is not None and isinstance(layer_types, list):
|
||||
num_swa = sum(
|
||||
1 for lt in layer_types[:num_layers] if "sliding" in lt.lower()
|
||||
)
|
||||
args.num_swa_layers = num_swa
|
||||
elif (
|
||||
getattr(getattr(model_config, "hf_config", None), "model_type", None)
|
||||
== "gemma2"
|
||||
):
|
||||
# Gemma 2 alternates sliding window: even layers (0, 2, ...) are sliding
|
||||
args.num_swa_layers = (num_layers + 1) // 2
|
||||
else:
|
||||
# Full SWA model (e.g. Mistral, Qwen2.5 with sliding_window)
|
||||
args.num_swa_layers = num_layers
|
||||
else:
|
||||
args.num_swa_layers = 0
|
||||
|
||||
return args
|
||||
|
||||
|
||||
class AttentionQuantizationConfigParser(Parser):
|
||||
"""
|
||||
Parses quantization configuration for attention layers.
|
||||
@@ -378,6 +472,7 @@ class AttentionQuantizationConfigParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Override weight_byte_size based on the active quantization method."""
|
||||
cfg = vllm_config.quant_config
|
||||
|
||||
if cfg is None:
|
||||
@@ -401,6 +496,7 @@ class AttentionDetectionParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Raise InvalidComponent for MLA models so AttentionMetrics is skipped."""
|
||||
if vllm_config.model_config.is_deepseek_mla:
|
||||
raise InvalidComponent(
|
||||
"Model uses MLA attention; use MLAAttentionMetrics instead"
|
||||
@@ -425,53 +521,129 @@ class AttentionMetrics(ComponentMetrics):
|
||||
# From BaseConfig Parser, overridden by AttentionQuantizationConfigParser
|
||||
weight_byte_size: int | float = Field(..., gt=0)
|
||||
|
||||
# TODO: discern cases where we have mixture of different attention layer types
|
||||
# such as SWA, MLA, etc.
|
||||
# From SlidingWindowAttentionParser
|
||||
sliding_window: int | None = Field(None)
|
||||
num_swa_layers: int = Field(0, ge=0)
|
||||
|
||||
@classmethod
|
||||
def component_type(cls) -> str:
|
||||
"""Return the component registry key for standard (non-MLA) attention."""
|
||||
return "attn"
|
||||
|
||||
@classmethod
|
||||
def get_parser(cls) -> ParserChain:
|
||||
"""Return the parser chain for AttentionMetrics.
|
||||
|
||||
Parsers run in order: MLA detection guard, base model config,
|
||||
attention-specific config, sliding-window breakdown, quantization
|
||||
weight-size override.
|
||||
"""
|
||||
return ParserChain(
|
||||
AttentionDetectionParser(),
|
||||
BaseConfigParser(),
|
||||
BaseAttentionConfigParser(),
|
||||
SlidingWindowAttentionParser(),
|
||||
AttentionQuantizationConfigParser(),
|
||||
)
|
||||
|
||||
def _layer_counts(self, per_gpu: bool) -> tuple[float, float, float]:
|
||||
"""Return ``(L, L_full, L_swa)``, the layer counts to bill for.
|
||||
|
||||
Per-GPU counts divide each global count by ``pp_size`` independently.
|
||||
Deriving the sublayer split after reducing ``L`` instead rounds a
|
||||
minority sublayer away: a 6-layer model with 1 sliding-window layer at
|
||||
``pp_size=2`` yields ``round(3 * 1 / 6) == 0``, billing all three layers
|
||||
as full attention. Dividing each count separately keeps
|
||||
``L_full + L_swa == L`` and keeps the per-GPU values summing back to the
|
||||
whole model across stages.
|
||||
|
||||
The counts are fractional because these metrics are built from config
|
||||
alone (see ``from_vllm_config``); no pipeline rank is available, so a
|
||||
per-GPU figure is the average stage rather than a specific one. A model
|
||||
whose layers do not divide evenly across stages has no single integer
|
||||
answer here.
|
||||
"""
|
||||
L = float(self.num_hidden_layers)
|
||||
L_swa = float(self.num_swa_layers)
|
||||
L_full = L - L_swa
|
||||
|
||||
if per_gpu:
|
||||
L /= self.pp_size
|
||||
L_swa /= self.pp_size
|
||||
L_full /= self.pp_size
|
||||
|
||||
return L, L_full, L_swa
|
||||
|
||||
def get_num_flops_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]:
|
||||
L, D, q, kv, d = (
|
||||
self.num_hidden_layers,
|
||||
"""Compute FLOPs breakdown for attention layers.
|
||||
|
||||
Accounts for hybrid models by splitting total layers into full-attention
|
||||
(L_full) and sliding-window (L_swa) sublayers. See ``_layer_counts`` for
|
||||
how those are scaled under pipeline parallelism.
|
||||
|
||||
Args:
|
||||
ctx: Execution context describing the current batch.
|
||||
per_gpu: If True, scale counts down by tensor/pipeline parallelism.
|
||||
|
||||
Returns:
|
||||
Dict with keys: qkv_proj, out_proj, attn_qk, attn_av.
|
||||
"""
|
||||
D, q, kv, d = (
|
||||
self.hidden_size,
|
||||
self.num_attention_heads,
|
||||
self.num_key_value_heads,
|
||||
self.head_dim,
|
||||
)
|
||||
T = ctx.total_num_tokens()
|
||||
TC = ctx.total_token_context_product()
|
||||
TC_full = ctx.total_token_context_product()
|
||||
|
||||
L, L_full, L_swa = self._layer_counts(per_gpu)
|
||||
|
||||
if per_gpu:
|
||||
L //= self.pp_size
|
||||
# tensor parallel along heads
|
||||
q = max(1, q // self.tp_size)
|
||||
kv = max(1, kv // self.tp_size)
|
||||
|
||||
return {
|
||||
"qkv_proj": 2 * T * D * (q + 2 * kv) * d * L,
|
||||
"attn_qk": 2 * q * TC * d * L,
|
||||
"attn_av": 2 * q * TC * d * L,
|
||||
"out_proj": 2 * T * D * q * d * L,
|
||||
flops = {
|
||||
"qkv_proj": int(2 * T * D * (q + 2 * kv) * d * L),
|
||||
"out_proj": int(2 * T * D * q * d * L),
|
||||
}
|
||||
|
||||
attn_qk = 0.0
|
||||
attn_av = 0.0
|
||||
if L_full > 0:
|
||||
attn_qk += 2 * q * TC_full * d * L_full
|
||||
attn_av += 2 * q * TC_full * d * L_full
|
||||
|
||||
if L_swa > 0 and self.sliding_window is not None:
|
||||
TC_swa = ctx.total_token_context_product(self.sliding_window)
|
||||
attn_qk += 2 * q * TC_swa * d * L_swa
|
||||
attn_av += 2 * q * TC_swa * d * L_swa
|
||||
|
||||
flops["attn_qk"] = int(attn_qk)
|
||||
flops["attn_av"] = int(attn_av)
|
||||
|
||||
return flops
|
||||
|
||||
def get_read_bytes_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]:
|
||||
L, D, q, kv, d = (
|
||||
self.num_hidden_layers,
|
||||
"""Compute read-memory-traffic breakdown for attention layers.
|
||||
|
||||
Splits layers into full-attention and sliding-window sublayers using the
|
||||
same counts as get_num_flops_breakdown; see ``_layer_counts``.
|
||||
|
||||
Args:
|
||||
ctx: Execution context describing the current batch.
|
||||
per_gpu: If True, scale counts down by tensor/pipeline parallelism.
|
||||
|
||||
Returns:
|
||||
Dict with keys: qkv_input, qkv_weight, attn_input (conditional),
|
||||
out_input, out_weight.
|
||||
"""
|
||||
D, q, kv, d = (
|
||||
self.hidden_size,
|
||||
self.num_attention_heads,
|
||||
self.num_key_value_heads,
|
||||
@@ -479,35 +651,63 @@ class AttentionMetrics(ComponentMetrics):
|
||||
)
|
||||
T = ctx.total_num_tokens()
|
||||
|
||||
L, L_full, L_swa = self._layer_counts(per_gpu)
|
||||
|
||||
if per_gpu:
|
||||
L //= self.pp_size
|
||||
# tensor parallel along heads
|
||||
q = max(1, q // self.tp_size)
|
||||
kv = max(1, kv // self.tp_size)
|
||||
|
||||
read_bytes = {}
|
||||
|
||||
read_bytes["qkv_input"] = T * D * self.activation_byte_size * L
|
||||
read_bytes["qkv_input"] = int(T * D * self.activation_byte_size * L)
|
||||
read_bytes["qkv_weight"] = int(D * (q + 2 * kv) * d * self.weight_byte_size * L)
|
||||
|
||||
# Attention input reads differ between prefill and decode
|
||||
# Prefill: read Q, K, V activations (all in activation_byte_size)
|
||||
if ctx.prefill_num_tokens > 0:
|
||||
read_bytes["attn_input"] = (
|
||||
(ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv)
|
||||
* d
|
||||
* self.activation_byte_size
|
||||
* L
|
||||
)
|
||||
attn_input = 0.0
|
||||
|
||||
# Decode: read Q activations + read K, V from cache (in cache_byte_size)
|
||||
if ctx.decode_num_tokens > 0:
|
||||
read_bytes["attn_input"] = read_bytes.get("attn_input", 0) + (
|
||||
ctx.decode_num_tokens * q * d * self.activation_byte_size * L
|
||||
+ 2 * ctx.decode_context_len * kv * d * self.cache_byte_size * L
|
||||
)
|
||||
# Full attention layers
|
||||
if L_full > 0:
|
||||
if ctx.prefill_num_tokens > 0:
|
||||
attn_input += (
|
||||
(ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv)
|
||||
* d
|
||||
* self.activation_byte_size
|
||||
* L_full
|
||||
)
|
||||
if ctx.decode_num_tokens > 0:
|
||||
attn_input += (
|
||||
ctx.decode_num_tokens * q * d * self.activation_byte_size * L_full
|
||||
+ 2
|
||||
* ctx.decode_context_len
|
||||
* kv
|
||||
* d
|
||||
* self.cache_byte_size
|
||||
* L_full
|
||||
)
|
||||
|
||||
read_bytes["out_input"] = T * q * d * self.activation_byte_size * L
|
||||
# Sliding window attention layers
|
||||
if L_swa > 0:
|
||||
sw = self.sliding_window
|
||||
prefill_sw_len = ctx.prefill_context_len_for_window(sw)
|
||||
decode_sw_len = ctx.decode_context_len_for_window(sw)
|
||||
|
||||
if ctx.prefill_num_tokens > 0:
|
||||
attn_input += (
|
||||
(ctx.prefill_num_tokens * q + 2 * prefill_sw_len * kv)
|
||||
* d
|
||||
* self.activation_byte_size
|
||||
* L_swa
|
||||
)
|
||||
if ctx.decode_num_tokens > 0:
|
||||
attn_input += (
|
||||
ctx.decode_num_tokens * q * d * self.activation_byte_size * L_swa
|
||||
+ 2 * decode_sw_len * kv * d * self.cache_byte_size * L_swa
|
||||
)
|
||||
|
||||
if ctx.prefill_num_tokens > 0 or ctx.decode_num_tokens > 0:
|
||||
read_bytes["attn_input"] = int(attn_input)
|
||||
|
||||
read_bytes["out_input"] = int(T * q * d * self.activation_byte_size * L)
|
||||
read_bytes["out_weight"] = int(q * d * D * self.weight_byte_size * L)
|
||||
|
||||
return read_bytes
|
||||
@@ -515,9 +715,13 @@ class AttentionMetrics(ComponentMetrics):
|
||||
def get_write_bytes_breakdown(
|
||||
self, ctx: ExecutionContext, per_gpu: bool = True
|
||||
) -> dict[str, int]:
|
||||
"""Calculate write memory traffic for attention layers."""
|
||||
L, D, q, kv, d = (
|
||||
self.num_hidden_layers,
|
||||
"""Calculate write memory traffic for attention layers.
|
||||
|
||||
Writes do not depend on the attention window, so only the total layer
|
||||
count is needed; it comes from ``_layer_counts`` so all three breakdowns
|
||||
bill the same number of layers.
|
||||
"""
|
||||
D, q, kv, d = (
|
||||
self.hidden_size,
|
||||
self.num_attention_heads,
|
||||
self.num_key_value_heads,
|
||||
@@ -525,16 +729,17 @@ class AttentionMetrics(ComponentMetrics):
|
||||
)
|
||||
T = ctx.total_num_tokens()
|
||||
|
||||
L, _, _ = self._layer_counts(per_gpu)
|
||||
|
||||
if per_gpu:
|
||||
L //= self.pp_size
|
||||
# tensor parallel along heads
|
||||
q = max(1, q // self.tp_size)
|
||||
kv = max(1, kv // self.tp_size)
|
||||
|
||||
return {
|
||||
"qkv_output": T * (q + 2 * kv) * d * self.activation_byte_size * L,
|
||||
"kv_cache": 2 * T * kv * d * self.cache_byte_size * L,
|
||||
"out_output": T * D * self.activation_byte_size * L,
|
||||
"qkv_output": int(T * (q + 2 * kv) * d * self.activation_byte_size * L),
|
||||
"kv_cache": int(2 * T * kv * d * self.cache_byte_size * L),
|
||||
"out_output": int(T * D * self.activation_byte_size * L),
|
||||
}
|
||||
|
||||
|
||||
@@ -549,6 +754,7 @@ class MLADetectionParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Raise InvalidComponent if the model does not use MLA attention."""
|
||||
if not vllm_config.model_config.is_deepseek_mla:
|
||||
raise InvalidComponent("Model does not use MLA attention")
|
||||
return args
|
||||
@@ -562,6 +768,7 @@ class MLAConfigParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Parse MLA-specific compression dimensions and KV-cache dtype byte size."""
|
||||
model_config = vllm_config.model_config
|
||||
cfg = model_config.hf_text_config
|
||||
|
||||
@@ -616,10 +823,16 @@ class MLAAttentionMetrics(ComponentMetrics):
|
||||
|
||||
@classmethod
|
||||
def component_type(cls) -> str:
|
||||
"""Return the component registry key for MLA attention."""
|
||||
return "mla_attn"
|
||||
|
||||
@classmethod
|
||||
def get_parser(cls) -> ParserChain:
|
||||
"""Return the parser chain for MLAAttentionMetrics.
|
||||
|
||||
Parsers run in order: MLA detection guard, base model config,
|
||||
MLA-specific config, quantization weight-size override.
|
||||
"""
|
||||
return ParserChain(
|
||||
MLADetectionParser(),
|
||||
BaseConfigParser(),
|
||||
@@ -923,6 +1136,7 @@ class FfnQuantizationConfigParser(Parser):
|
||||
"""
|
||||
|
||||
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
|
||||
"""Override weight_byte_size based on the active FFN quantization method."""
|
||||
cfg = vllm_config.quant_config
|
||||
|
||||
if cfg is None:
|
||||
@@ -980,10 +1194,16 @@ class FfnMetrics(ComponentMetrics):
|
||||
|
||||
@classmethod
|
||||
def component_type(cls) -> str:
|
||||
"""Return the component registry key for FFN layers."""
|
||||
return "ffn"
|
||||
|
||||
@classmethod
|
||||
def get_parser(cls) -> ParserChain:
|
||||
"""Return the parser chain for FfnMetrics.
|
||||
|
||||
Parsers run in order: base model config, FFN parallelism, base FFN
|
||||
config, MoE interleave step, MoE layer frequency, quantization override.
|
||||
"""
|
||||
return ParserChain(
|
||||
BaseConfigParser(),
|
||||
FfnParallelParser(),
|
||||
@@ -1207,10 +1427,12 @@ class UnembedMetrics(ComponentMetrics):
|
||||
|
||||
@classmethod
|
||||
def component_type(cls) -> str:
|
||||
"""Return the component registry key for the unembedding (LM head) layer."""
|
||||
return "unembed"
|
||||
|
||||
@classmethod
|
||||
def get_parser(cls) -> ParserChain:
|
||||
"""Return the parser chain for UnembedMetrics (base model config only)."""
|
||||
return ParserChain(
|
||||
BaseConfigParser(),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user