[V1][Metrics] Support Sliding Window Attention (SWA) and hybrid layers in MFU/MBU estimation (#55624)

Signed-off-by: Thillai Chithambaram <thillaichithambaram.a@gmail.com>
Co-authored-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Thillai Chithambaram
2026-09-14 16:04:55 +00:00
committed by GitHub
co-authored by Mark McLoughlin
parent b7e8dd8f37
commit 435c96f9db
2 changed files with 640 additions and 48 deletions
+372 -2
View File
@@ -8,6 +8,7 @@ import types
from types import SimpleNamespace
import pytest
from transformers import Gemma2Config, MistralConfig
from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config
from transformers.models.llama4.configuration_llama4 import (
Llama4Config,
@@ -31,6 +32,7 @@ from vllm.v1.metrics.perf import (
MLAAttentionMetrics,
ModelMetrics,
ParsedArgs,
SlidingWindowAttentionParser,
UnembedMetrics,
)
@@ -38,7 +40,8 @@ from vllm.v1.metrics.perf import (
class MockModelConfig:
"""Mock ModelConfig that implements the getter methods used by parsers."""
def __init__(self, hf_config, dtype):
def __init__(self, hf_config, dtype, disable_sliding_window=False):
"""Initialize MockModelConfig with HF config and options."""
self.hf_config = hf_config
self.hf_text_config = get_hf_text_config(hf_config)
convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get(
@@ -49,8 +52,10 @@ class MockModelConfig:
).convert()
self.dtype = dtype
self.is_attention_free = False
self.disable_sliding_window = disable_sliding_window
def __getattr__(self, name):
"""Delegate attribute lookups to ModelConfig."""
# 1. Check if ModelConfig actually has this attribute
if not hasattr(ModelConfig, name):
raise AttributeError(
@@ -86,9 +91,13 @@ def create_mock_vllm_config(
tensor_parallel_size=1,
pipeline_parallel_size=1,
enable_expert_parallel=False,
disable_sliding_window=False,
) -> SimpleNamespace:
"""Create a mock VllmConfig object for unit tests."""
vllm_config = SimpleNamespace()
vllm_config.model_config = MockModelConfig(hf_config, model_dtype)
vllm_config.model_config = MockModelConfig(
hf_config, model_dtype, disable_sliding_window=disable_sliding_window
)
vllm_config.cache_config = SimpleNamespace()
vllm_config.cache_config.cache_dtype = cache_dtype
@@ -1336,3 +1345,364 @@ def test_mla_attention_scaling_with_layers():
assert double_metrics.get_num_flops(ctx) == 2 * base_metrics.get_num_flops(ctx)
assert double_metrics.get_read_bytes(ctx) == 2 * base_metrics.get_read_bytes(ctx)
assert double_metrics.get_write_bytes(ctx) == 2 * base_metrics.get_write_bytes(ctx)
#### Sliding Window Attention Tests ####
def test_sliding_window_parser_no_swa():
"""Test parser defaults to no SWA when sliding_window is unset."""
hf_config = Qwen3Config(
hidden_size=2048,
num_attention_heads=16,
num_hidden_layers=12,
)
vllm_config = create_mock_vllm_config(hf_config)
parser = SlidingWindowAttentionParser()
args = ParsedArgs()
args.num_hidden_layers = 12
result = parser.parse(args, vllm_config)
assert result.sliding_window is None
assert result.num_swa_layers == 0
def test_sliding_window_parser_uniform_swa():
"""Test SlidingWindowAttentionParser for full SWA model (e.g. Mistral)."""
hf_config = MistralConfig(
hidden_size=4096,
num_attention_heads=32,
num_key_value_heads=8,
num_hidden_layers=32,
sliding_window=4096,
)
vllm_config = create_mock_vllm_config(hf_config)
parser = SlidingWindowAttentionParser()
args = ParsedArgs()
args.num_hidden_layers = 32
result = parser.parse(args, vllm_config)
assert result.sliding_window == 4096
assert result.num_swa_layers == 32
def test_sliding_window_parser_explicit_layer_types():
"""Test SlidingWindowAttentionParser with explicit layer_types list."""
hf_config = Qwen3Config(
hidden_size=2048,
num_attention_heads=16,
num_hidden_layers=6,
)
hf_config.sliding_window = 2048
hf_config.layer_types = [
"sliding_attention",
"full_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"full_attention",
]
vllm_config = create_mock_vllm_config(hf_config)
parser = SlidingWindowAttentionParser()
args = ParsedArgs()
args.num_hidden_layers = 6
result = parser.parse(args, vllm_config)
assert result.sliding_window == 2048
assert result.num_swa_layers == 3
def test_sliding_window_parser_gemma2():
"""Test SlidingWindowAttentionParser detects alternating SWA for Gemma 2."""
hf_config = Gemma2Config(
hidden_size=2304,
num_attention_heads=8,
num_key_value_heads=4,
num_hidden_layers=26,
sliding_window=4096,
)
vllm_config = create_mock_vllm_config(hf_config)
parser = SlidingWindowAttentionParser()
args = ParsedArgs()
args.num_hidden_layers = 26
result = parser.parse(args, vllm_config)
assert result.sliding_window == 4096
# Gemma 2 alternates: even layers are sliding window (26 layers -> 13 SWA)
assert result.num_swa_layers == 13
def test_sliding_window_parser_disabled():
"""Test SlidingWindowAttentionParser respects disable_sliding_window."""
hf_config = MistralConfig(
hidden_size=4096,
num_attention_heads=32,
num_key_value_heads=8,
num_hidden_layers=32,
sliding_window=4096,
)
vllm_config = create_mock_vllm_config(hf_config, disable_sliding_window=True)
parser = SlidingWindowAttentionParser()
args = ParsedArgs()
args.num_hidden_layers = 32
result = parser.parse(args, vllm_config)
assert result.sliding_window is None
assert result.num_swa_layers == 0
def test_attention_metrics_sliding_window_flops_bounds():
"""Test that SWA clamps attention core FLOPs when context_len > sliding_window."""
sw = 1024
dense_config = MistralConfig(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=8,
sliding_window=None,
)
swa_config = MistralConfig(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=8,
sliding_window=sw,
)
dense_vllm = create_mock_vllm_config(dense_config)
swa_vllm = create_mock_vllm_config(swa_config)
dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm)
swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm)
# 1. When context <= sliding_window: FLOPs should be identical
short_ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=512, is_prefill=False
)
assert swa_metrics.get_num_flops(short_ctx) == dense_metrics.get_num_flops(
short_ctx
)
# 2. When context > sliding_window: FLOPs for SWA must be strictly smaller
long_ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=8192, is_prefill=False
)
dense_flops = dense_metrics.get_num_flops_breakdown(long_ctx)
swa_flops = swa_metrics.get_num_flops_breakdown(long_ctx)
# Projections (qkv_proj, out_proj) are unaffected by sliding window
assert swa_flops["qkv_proj"] == dense_flops["qkv_proj"]
assert swa_flops["out_proj"] == dense_flops["out_proj"]
# Attention core FLOPs are clamped: 8192 vs 1024 (8x difference)
assert swa_flops["attn_qk"] < dense_flops["attn_qk"]
assert swa_flops["attn_av"] < dense_flops["attn_av"]
assert swa_flops["attn_qk"] * 8 == dense_flops["attn_qk"]
assert swa_flops["attn_av"] * 8 == dense_flops["attn_av"]
def test_attention_metrics_sliding_window_read_bytes():
"""Test that SWA bounds decode KV-cache read bytes to sliding_window."""
sw = 2048
dense_config = MistralConfig(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=4,
num_hidden_layers=4,
sliding_window=None,
)
swa_config = MistralConfig(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=4,
num_hidden_layers=4,
sliding_window=sw,
)
dense_vllm = create_mock_vllm_config(dense_config)
swa_vllm = create_mock_vllm_config(swa_config)
dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm)
swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm)
# Long decode context: 8192 tokens
ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=8192, is_prefill=False
)
dense_reads = dense_metrics.get_read_bytes_breakdown(ctx)
swa_reads = swa_metrics.get_read_bytes_breakdown(ctx)
# Attention input read traffic is lower for SWA
assert swa_reads["attn_input"] < dense_reads["attn_input"]
# Weight reads and projection inputs are identical
assert swa_reads["qkv_weight"] == dense_reads["qkv_weight"]
assert swa_reads["out_weight"] == dense_reads["out_weight"]
assert swa_reads["qkv_input"] == dense_reads["qkv_input"]
def test_attention_metrics_hybrid_alternating_layers():
"""Test hybrid model where half layers are full attention and half are SWA."""
sw = 1024
hybrid_config = Qwen3Config(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=4,
)
hybrid_config.sliding_window = sw
hybrid_config.layer_types = [
"sliding_attention",
"full_attention",
"sliding_attention",
"full_attention",
]
vllm_config = create_mock_vllm_config(hybrid_config)
metrics = AttentionMetrics.from_vllm_config(vllm_config)
assert metrics.num_swa_layers == 2
assert metrics.sliding_window == sw
ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=4096, is_prefill=False
)
flops = metrics.get_num_flops_breakdown(ctx)
# 2 full layers attend to 4096; 2 SWA layers attend to 1024
# Expected TC total = 2 * 4096 + 2 * 1024 = 8192 + 2048 = 10240
# Formula: 2 * q * d * (L_full * 4096 + L_swa * 1024)
q = metrics.num_attention_heads
d = metrics.head_dim
expected_core_flops = 2 * q * d * (2 * 4096 + 2 * 1024)
assert flops["attn_qk"] == expected_core_flops
assert flops["attn_av"] == expected_core_flops
def test_attention_metrics_sliding_window_per_gpu():
"""Test that SWA metrics scale properly with tensor and pipeline parallelism."""
sw = 1024
hf_config = MistralConfig(
hidden_size=4096,
num_attention_heads=32,
num_key_value_heads=8,
num_hidden_layers=16,
sliding_window=sw,
)
single_vllm = create_mock_vllm_config(hf_config)
tp_vllm = create_mock_vllm_config(hf_config, tensor_parallel_size=2)
pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2)
single_metrics = AttentionMetrics.from_vllm_config(single_vllm)
tp_metrics = AttentionMetrics.from_vllm_config(tp_vllm)
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=4096, is_prefill=False
)
# Both TP=2 and PP=2 should halve the per-GPU attention core FLOPs
single_flops = single_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
tp_flops = tp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
assert tp_flops["attn_qk"] == single_flops["attn_qk"] // 2
assert pp_flops["attn_qk"] == single_flops["attn_qk"] // 2
def test_attention_metrics_hybrid_sliding_window_per_gpu():
"""Test that hybrid SWA models preserve L_full + L_swa == L under PP."""
sw = 1024
hybrid_config = Gemma2Config(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=30,
sliding_window=sw,
)
pp_vllm = create_mock_vllm_config(hybrid_config, pipeline_parallel_size=2)
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=4096, is_prefill=False
)
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
pp_read_bytes = pp_metrics.get_read_bytes_breakdown(ctx, per_gpu=True)
# Under PP=2, each global count is halved: 30 layers -> 15 per GPU, and the
# 15 SWA / 15 full split becomes 7.5 each. Gemma 2 alternates, so an even
# split is the correct average stage; deriving it after reducing L instead
# produced a lopsided 8/7.
q = pp_metrics.num_attention_heads
d = pp_metrics.head_dim
expected_pp_attn = int(2 * q * d * (7.5 * 4096 + 7.5 * sw))
assert pp_flops["attn_qk"] == expected_pp_attn
assert pp_flops["attn_av"] == expected_pp_attn
assert "attn_input" in pp_read_bytes
def test_attention_metrics_minority_swa_survives_pipeline_parallelism():
"""A minority SWA sublayer must not round away under PP.
Regression test for a 6-layer model with a single sliding-window layer at
pp_size=2: deriving the split after reducing L gave
round(3 * 1 / 6) == 0, billing all three per-GPU layers as full attention
and dropping the sliding-window contribution entirely.
"""
sw = 512
context_len = 4096
hf_config = Qwen3Config(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=6,
)
hf_config.sliding_window = sw
hf_config.layer_types = ["sliding_attention"] + ["full_attention"] * 5
pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2)
pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm)
assert pp_metrics.num_swa_layers == 1
ctx = ExecutionContext.from_single_request(
num_tokens=1, context_len=context_len, is_prefill=False
)
pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True)
# 5 full and 1 SWA layer, each halved by PP=2: 2.5 full and 0.5 SWA.
q = pp_metrics.num_attention_heads
d = pp_metrics.head_dim
expected = int(2 * q * d * (2.5 * context_len + 0.5 * sw))
assert pp_flops["attn_qk"] == expected
assert pp_flops["attn_av"] == expected
# The sliding-window layer must change the result, i.e. it is not being
# billed at the full context length.
all_full = int(2 * q * d * 3 * context_len)
assert pp_flops["attn_qk"] < all_full
def test_attention_metrics_per_gpu_layers_sum_to_whole_model():
"""Per-GPU counts times pp_size must account for every layer.
Flooring the layer count per stage silently drops the remainder, so a model
whose layers do not divide evenly across stages under-reports its FLOPs.
"""
hf_config = Qwen3Config(
hidden_size=2048,
num_attention_heads=16,
num_key_value_heads=16,
num_hidden_layers=7,
)
pp_size = 2
vllm_config = create_mock_vllm_config(hf_config, pipeline_parallel_size=pp_size)
metrics = AttentionMetrics.from_vllm_config(vllm_config)
ctx = ExecutionContext.from_single_request(
num_tokens=8, context_len=1024, is_prefill=True
)
per_gpu = metrics.get_num_flops_breakdown(ctx, per_gpu=True)
whole = metrics.get_num_flops_breakdown(ctx, per_gpu=False)
for key, value in whole.items():
assert per_gpu[key] * pp_size == pytest.approx(value, rel=1e-9)
+268 -46
View File
@@ -10,7 +10,7 @@ import json
import time
from abc import ABC, abstractmethod
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field
from typing import Any, Protocol
import prometheus_client
@@ -124,8 +124,12 @@ class ExecutionContext:
decode_context_len: int = 0 # sum of context_len for decode requests
decode_token_context_product: int = 0 # sum of (num_tokens * context_len)
# Per-request records: (num_tokens, context_len, is_prefill)
requests: list[tuple[int, int, bool]] = field(default_factory=list)
def add(self, num_tokens: int, context_len: int, is_prefill: bool) -> None:
"""Add a single request's statistics to this batch context."""
self.requests.append((num_tokens, context_len, is_prefill))
if is_prefill:
self.num_prefill_requests += 1
self.prefill_num_tokens += num_tokens
@@ -141,9 +145,40 @@ class ExecutionContext:
"""Total number of tokens across all requests in the batch."""
return self.prefill_num_tokens + self.decode_num_tokens
def total_token_context_product(self) -> int:
"""Total sum of (num_tokens * context_len) across all requests."""
return self.prefill_token_context_product + self.decode_token_context_product
def total_token_context_product(self, sliding_window: int | None = None) -> int:
"""Total sum of (num_tokens * context_len) across all requests.
If sliding_window is provided, context length per request is clamped to
sliding_window.
"""
if sliding_window is None:
return (
self.prefill_token_context_product + self.decode_token_context_product
)
return sum(
num_tokens * min(context_len, sliding_window)
for num_tokens, context_len, _ in self.requests
)
def prefill_context_len_for_window(self, sliding_window: int | None = None) -> int:
"""Sum of context_len for prefill requests, clamped to sliding_window if set."""
if sliding_window is None:
return self.prefill_context_len
return sum(
min(context_len, sliding_window)
for _, context_len, is_prefill in self.requests
if is_prefill
)
def decode_context_len_for_window(self, sliding_window: int | None = None) -> int:
"""Sum of context_len for decode requests, clamped to sliding_window if set."""
if sliding_window is None:
return self.decode_context_len
return sum(
min(context_len, sliding_window)
for _, context_len, is_prefill in self.requests
if not is_prefill
)
def num_logits_tokens(self) -> int:
"""Number of tokens that require logits computation (unembedding).
@@ -214,6 +249,7 @@ class ParserChain:
self.parsers.append(parser)
def parse(self, vllm_config: VllmConfig) -> ParsedArgs:
"""Apply all parsers in sequence and return the accumulated ParsedArgs."""
args = ParsedArgs()
for parser in self.parsers:
args = parser.parse(args, vllm_config)
@@ -234,7 +270,9 @@ class ComponentMetrics(BaseModel, ABC):
@classmethod
@abstractmethod
def component_type(cls) -> str: ...
def component_type(cls) -> str:
"""Return the unique string key identifying this component in the registry."""
...
@classmethod
@abstractmethod
@@ -273,17 +311,23 @@ class ComponentMetrics(BaseModel, ABC):
@abstractmethod
def get_num_flops_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]: ...
) -> dict[str, int]:
"""Return per-operation FLOPs breakdown for this component."""
...
@abstractmethod
def get_read_bytes_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]: ...
) -> dict[str, int]:
"""Return per-operation read-memory-traffic breakdown for this component."""
...
@abstractmethod
def get_write_bytes_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]: ...
) -> dict[str, int]:
"""Return per-operation write-memory-traffic breakdown for this component."""
...
def get_num_flops(self, ctx: ExecutionContext, per_gpu: bool = True) -> int:
return sum(self.get_num_flops_breakdown(ctx, per_gpu).values())
@@ -306,6 +350,7 @@ class BaseConfigParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Parse base model dimensions, dtype byte sizes, and parallelism config."""
model_config = vllm_config.model_config
args.vocab_size = model_config.get_vocab_size()
@@ -357,6 +402,7 @@ class BaseAttentionConfigParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Parse KV head count, head dimension, and KV-cache dtype byte size."""
model_config = vllm_config.model_config
args.num_key_value_heads = model_config.get_total_num_kv_heads()
@@ -371,6 +417,54 @@ class BaseAttentionConfigParser(Parser):
return args
class SlidingWindowAttentionParser(Parser):
"""
Parses sliding window attention configuration and layer breakdown.
Provides: sliding_window, num_swa_layers
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Determine sliding_window size and count of SWA layers.
Detects per-layer type lists (e.g. Llama-4), Gemma 2 alternating
pattern, and uniform SWA models (Mistral, Qwen2.5).
"""
model_config = vllm_config.model_config
sliding_window: int | None = None
if not getattr(model_config, "disable_sliding_window", False):
sliding_window = model_config.get_sliding_window()
if sliding_window is None and hasattr(vllm_config, "cache_config"):
sliding_window = getattr(
vllm_config.cache_config, "sliding_window", None
)
args.sliding_window = sliding_window
num_layers = args.num_hidden_layers
if sliding_window is not None and sliding_window > 0:
hf_text_config = getattr(model_config, "hf_text_config", None)
layer_types = getattr(hf_text_config, "layer_types", None)
if layer_types is not None and isinstance(layer_types, list):
num_swa = sum(
1 for lt in layer_types[:num_layers] if "sliding" in lt.lower()
)
args.num_swa_layers = num_swa
elif (
getattr(getattr(model_config, "hf_config", None), "model_type", None)
== "gemma2"
):
# Gemma 2 alternates sliding window: even layers (0, 2, ...) are sliding
args.num_swa_layers = (num_layers + 1) // 2
else:
# Full SWA model (e.g. Mistral, Qwen2.5 with sliding_window)
args.num_swa_layers = num_layers
else:
args.num_swa_layers = 0
return args
class AttentionQuantizationConfigParser(Parser):
"""
Parses quantization configuration for attention layers.
@@ -378,6 +472,7 @@ class AttentionQuantizationConfigParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Override weight_byte_size based on the active quantization method."""
cfg = vllm_config.quant_config
if cfg is None:
@@ -401,6 +496,7 @@ class AttentionDetectionParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Raise InvalidComponent for MLA models so AttentionMetrics is skipped."""
if vllm_config.model_config.is_deepseek_mla:
raise InvalidComponent(
"Model uses MLA attention; use MLAAttentionMetrics instead"
@@ -425,53 +521,129 @@ class AttentionMetrics(ComponentMetrics):
# From BaseConfig Parser, overridden by AttentionQuantizationConfigParser
weight_byte_size: int | float = Field(..., gt=0)
# TODO: discern cases where we have mixture of different attention layer types
# such as SWA, MLA, etc.
# From SlidingWindowAttentionParser
sliding_window: int | None = Field(None)
num_swa_layers: int = Field(0, ge=0)
@classmethod
def component_type(cls) -> str:
"""Return the component registry key for standard (non-MLA) attention."""
return "attn"
@classmethod
def get_parser(cls) -> ParserChain:
"""Return the parser chain for AttentionMetrics.
Parsers run in order: MLA detection guard, base model config,
attention-specific config, sliding-window breakdown, quantization
weight-size override.
"""
return ParserChain(
AttentionDetectionParser(),
BaseConfigParser(),
BaseAttentionConfigParser(),
SlidingWindowAttentionParser(),
AttentionQuantizationConfigParser(),
)
def _layer_counts(self, per_gpu: bool) -> tuple[float, float, float]:
"""Return ``(L, L_full, L_swa)``, the layer counts to bill for.
Per-GPU counts divide each global count by ``pp_size`` independently.
Deriving the sublayer split after reducing ``L`` instead rounds a
minority sublayer away: a 6-layer model with 1 sliding-window layer at
``pp_size=2`` yields ``round(3 * 1 / 6) == 0``, billing all three layers
as full attention. Dividing each count separately keeps
``L_full + L_swa == L`` and keeps the per-GPU values summing back to the
whole model across stages.
The counts are fractional because these metrics are built from config
alone (see ``from_vllm_config``); no pipeline rank is available, so a
per-GPU figure is the average stage rather than a specific one. A model
whose layers do not divide evenly across stages has no single integer
answer here.
"""
L = float(self.num_hidden_layers)
L_swa = float(self.num_swa_layers)
L_full = L - L_swa
if per_gpu:
L /= self.pp_size
L_swa /= self.pp_size
L_full /= self.pp_size
return L, L_full, L_swa
def get_num_flops_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]:
L, D, q, kv, d = (
self.num_hidden_layers,
"""Compute FLOPs breakdown for attention layers.
Accounts for hybrid models by splitting total layers into full-attention
(L_full) and sliding-window (L_swa) sublayers. See ``_layer_counts`` for
how those are scaled under pipeline parallelism.
Args:
ctx: Execution context describing the current batch.
per_gpu: If True, scale counts down by tensor/pipeline parallelism.
Returns:
Dict with keys: qkv_proj, out_proj, attn_qk, attn_av.
"""
D, q, kv, d = (
self.hidden_size,
self.num_attention_heads,
self.num_key_value_heads,
self.head_dim,
)
T = ctx.total_num_tokens()
TC = ctx.total_token_context_product()
TC_full = ctx.total_token_context_product()
L, L_full, L_swa = self._layer_counts(per_gpu)
if per_gpu:
L //= self.pp_size
# tensor parallel along heads
q = max(1, q // self.tp_size)
kv = max(1, kv // self.tp_size)
return {
"qkv_proj": 2 * T * D * (q + 2 * kv) * d * L,
"attn_qk": 2 * q * TC * d * L,
"attn_av": 2 * q * TC * d * L,
"out_proj": 2 * T * D * q * d * L,
flops = {
"qkv_proj": int(2 * T * D * (q + 2 * kv) * d * L),
"out_proj": int(2 * T * D * q * d * L),
}
attn_qk = 0.0
attn_av = 0.0
if L_full > 0:
attn_qk += 2 * q * TC_full * d * L_full
attn_av += 2 * q * TC_full * d * L_full
if L_swa > 0 and self.sliding_window is not None:
TC_swa = ctx.total_token_context_product(self.sliding_window)
attn_qk += 2 * q * TC_swa * d * L_swa
attn_av += 2 * q * TC_swa * d * L_swa
flops["attn_qk"] = int(attn_qk)
flops["attn_av"] = int(attn_av)
return flops
def get_read_bytes_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]:
L, D, q, kv, d = (
self.num_hidden_layers,
"""Compute read-memory-traffic breakdown for attention layers.
Splits layers into full-attention and sliding-window sublayers using the
same counts as get_num_flops_breakdown; see ``_layer_counts``.
Args:
ctx: Execution context describing the current batch.
per_gpu: If True, scale counts down by tensor/pipeline parallelism.
Returns:
Dict with keys: qkv_input, qkv_weight, attn_input (conditional),
out_input, out_weight.
"""
D, q, kv, d = (
self.hidden_size,
self.num_attention_heads,
self.num_key_value_heads,
@@ -479,35 +651,63 @@ class AttentionMetrics(ComponentMetrics):
)
T = ctx.total_num_tokens()
L, L_full, L_swa = self._layer_counts(per_gpu)
if per_gpu:
L //= self.pp_size
# tensor parallel along heads
q = max(1, q // self.tp_size)
kv = max(1, kv // self.tp_size)
read_bytes = {}
read_bytes["qkv_input"] = T * D * self.activation_byte_size * L
read_bytes["qkv_input"] = int(T * D * self.activation_byte_size * L)
read_bytes["qkv_weight"] = int(D * (q + 2 * kv) * d * self.weight_byte_size * L)
# Attention input reads differ between prefill and decode
# Prefill: read Q, K, V activations (all in activation_byte_size)
if ctx.prefill_num_tokens > 0:
read_bytes["attn_input"] = (
(ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv)
* d
* self.activation_byte_size
* L
)
attn_input = 0.0
# Decode: read Q activations + read K, V from cache (in cache_byte_size)
if ctx.decode_num_tokens > 0:
read_bytes["attn_input"] = read_bytes.get("attn_input", 0) + (
ctx.decode_num_tokens * q * d * self.activation_byte_size * L
+ 2 * ctx.decode_context_len * kv * d * self.cache_byte_size * L
)
# Full attention layers
if L_full > 0:
if ctx.prefill_num_tokens > 0:
attn_input += (
(ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv)
* d
* self.activation_byte_size
* L_full
)
if ctx.decode_num_tokens > 0:
attn_input += (
ctx.decode_num_tokens * q * d * self.activation_byte_size * L_full
+ 2
* ctx.decode_context_len
* kv
* d
* self.cache_byte_size
* L_full
)
read_bytes["out_input"] = T * q * d * self.activation_byte_size * L
# Sliding window attention layers
if L_swa > 0:
sw = self.sliding_window
prefill_sw_len = ctx.prefill_context_len_for_window(sw)
decode_sw_len = ctx.decode_context_len_for_window(sw)
if ctx.prefill_num_tokens > 0:
attn_input += (
(ctx.prefill_num_tokens * q + 2 * prefill_sw_len * kv)
* d
* self.activation_byte_size
* L_swa
)
if ctx.decode_num_tokens > 0:
attn_input += (
ctx.decode_num_tokens * q * d * self.activation_byte_size * L_swa
+ 2 * decode_sw_len * kv * d * self.cache_byte_size * L_swa
)
if ctx.prefill_num_tokens > 0 or ctx.decode_num_tokens > 0:
read_bytes["attn_input"] = int(attn_input)
read_bytes["out_input"] = int(T * q * d * self.activation_byte_size * L)
read_bytes["out_weight"] = int(q * d * D * self.weight_byte_size * L)
return read_bytes
@@ -515,9 +715,13 @@ class AttentionMetrics(ComponentMetrics):
def get_write_bytes_breakdown(
self, ctx: ExecutionContext, per_gpu: bool = True
) -> dict[str, int]:
"""Calculate write memory traffic for attention layers."""
L, D, q, kv, d = (
self.num_hidden_layers,
"""Calculate write memory traffic for attention layers.
Writes do not depend on the attention window, so only the total layer
count is needed; it comes from ``_layer_counts`` so all three breakdowns
bill the same number of layers.
"""
D, q, kv, d = (
self.hidden_size,
self.num_attention_heads,
self.num_key_value_heads,
@@ -525,16 +729,17 @@ class AttentionMetrics(ComponentMetrics):
)
T = ctx.total_num_tokens()
L, _, _ = self._layer_counts(per_gpu)
if per_gpu:
L //= self.pp_size
# tensor parallel along heads
q = max(1, q // self.tp_size)
kv = max(1, kv // self.tp_size)
return {
"qkv_output": T * (q + 2 * kv) * d * self.activation_byte_size * L,
"kv_cache": 2 * T * kv * d * self.cache_byte_size * L,
"out_output": T * D * self.activation_byte_size * L,
"qkv_output": int(T * (q + 2 * kv) * d * self.activation_byte_size * L),
"kv_cache": int(2 * T * kv * d * self.cache_byte_size * L),
"out_output": int(T * D * self.activation_byte_size * L),
}
@@ -549,6 +754,7 @@ class MLADetectionParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Raise InvalidComponent if the model does not use MLA attention."""
if not vllm_config.model_config.is_deepseek_mla:
raise InvalidComponent("Model does not use MLA attention")
return args
@@ -562,6 +768,7 @@ class MLAConfigParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Parse MLA-specific compression dimensions and KV-cache dtype byte size."""
model_config = vllm_config.model_config
cfg = model_config.hf_text_config
@@ -616,10 +823,16 @@ class MLAAttentionMetrics(ComponentMetrics):
@classmethod
def component_type(cls) -> str:
"""Return the component registry key for MLA attention."""
return "mla_attn"
@classmethod
def get_parser(cls) -> ParserChain:
"""Return the parser chain for MLAAttentionMetrics.
Parsers run in order: MLA detection guard, base model config,
MLA-specific config, quantization weight-size override.
"""
return ParserChain(
MLADetectionParser(),
BaseConfigParser(),
@@ -923,6 +1136,7 @@ class FfnQuantizationConfigParser(Parser):
"""
def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs:
"""Override weight_byte_size based on the active FFN quantization method."""
cfg = vllm_config.quant_config
if cfg is None:
@@ -980,10 +1194,16 @@ class FfnMetrics(ComponentMetrics):
@classmethod
def component_type(cls) -> str:
"""Return the component registry key for FFN layers."""
return "ffn"
@classmethod
def get_parser(cls) -> ParserChain:
"""Return the parser chain for FfnMetrics.
Parsers run in order: base model config, FFN parallelism, base FFN
config, MoE interleave step, MoE layer frequency, quantization override.
"""
return ParserChain(
BaseConfigParser(),
FfnParallelParser(),
@@ -1207,10 +1427,12 @@ class UnembedMetrics(ComponentMetrics):
@classmethod
def component_type(cls) -> str:
"""Return the component registry key for the unembedding (LM head) layer."""
return "unembed"
@classmethod
def get_parser(cls) -> ParserChain:
"""Return the parser chain for UnembedMetrics (base model config only)."""
return ParserChain(
BaseConfigParser(),
)