From 435c96f9dbdd29258cb8e0f433c5b54a00cf6b16 Mon Sep 17 00:00:00 2001 From: Thillai Chithambaram <79466435+thillai-c@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:04:55 -0400 Subject: [PATCH] [V1][Metrics] Support Sliding Window Attention (SWA) and hybrid layers in MFU/MBU estimation (#55624) Signed-off-by: Thillai Chithambaram Co-authored-by: Mark McLoughlin --- tests/v1/metrics/test_perf_metrics.py | 374 +++++++++++++++++++++++++- vllm/v1/metrics/perf.py | 314 +++++++++++++++++---- 2 files changed, 640 insertions(+), 48 deletions(-) diff --git a/tests/v1/metrics/test_perf_metrics.py b/tests/v1/metrics/test_perf_metrics.py index ab30f1bb9e..c94ec1b701 100644 --- a/tests/v1/metrics/test_perf_metrics.py +++ b/tests/v1/metrics/test_perf_metrics.py @@ -8,6 +8,7 @@ import types from types import SimpleNamespace import pytest +from transformers import Gemma2Config, MistralConfig from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config from transformers.models.llama4.configuration_llama4 import ( Llama4Config, @@ -31,6 +32,7 @@ from vllm.v1.metrics.perf import ( MLAAttentionMetrics, ModelMetrics, ParsedArgs, + SlidingWindowAttentionParser, UnembedMetrics, ) @@ -38,7 +40,8 @@ from vllm.v1.metrics.perf import ( class MockModelConfig: """Mock ModelConfig that implements the getter methods used by parsers.""" - def __init__(self, hf_config, dtype): + def __init__(self, hf_config, dtype, disable_sliding_window=False): + """Initialize MockModelConfig with HF config and options.""" self.hf_config = hf_config self.hf_text_config = get_hf_text_config(hf_config) convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get( @@ -49,8 +52,10 @@ class MockModelConfig: ).convert() self.dtype = dtype self.is_attention_free = False + self.disable_sliding_window = disable_sliding_window def __getattr__(self, name): + """Delegate attribute lookups to ModelConfig.""" # 1. Check if ModelConfig actually has this attribute if not hasattr(ModelConfig, name): raise AttributeError( @@ -86,9 +91,13 @@ def create_mock_vllm_config( tensor_parallel_size=1, pipeline_parallel_size=1, enable_expert_parallel=False, + disable_sliding_window=False, ) -> SimpleNamespace: + """Create a mock VllmConfig object for unit tests.""" vllm_config = SimpleNamespace() - vllm_config.model_config = MockModelConfig(hf_config, model_dtype) + vllm_config.model_config = MockModelConfig( + hf_config, model_dtype, disable_sliding_window=disable_sliding_window + ) vllm_config.cache_config = SimpleNamespace() vllm_config.cache_config.cache_dtype = cache_dtype @@ -1336,3 +1345,364 @@ def test_mla_attention_scaling_with_layers(): assert double_metrics.get_num_flops(ctx) == 2 * base_metrics.get_num_flops(ctx) assert double_metrics.get_read_bytes(ctx) == 2 * base_metrics.get_read_bytes(ctx) assert double_metrics.get_write_bytes(ctx) == 2 * base_metrics.get_write_bytes(ctx) + + +#### Sliding Window Attention Tests #### + + +def test_sliding_window_parser_no_swa(): + """Test parser defaults to no SWA when sliding_window is unset.""" + hf_config = Qwen3Config( + hidden_size=2048, + num_attention_heads=16, + num_hidden_layers=12, + ) + vllm_config = create_mock_vllm_config(hf_config) + parser = SlidingWindowAttentionParser() + args = ParsedArgs() + args.num_hidden_layers = 12 + result = parser.parse(args, vllm_config) + + assert result.sliding_window is None + assert result.num_swa_layers == 0 + + +def test_sliding_window_parser_uniform_swa(): + """Test SlidingWindowAttentionParser for full SWA model (e.g. Mistral).""" + hf_config = MistralConfig( + hidden_size=4096, + num_attention_heads=32, + num_key_value_heads=8, + num_hidden_layers=32, + sliding_window=4096, + ) + vllm_config = create_mock_vllm_config(hf_config) + parser = SlidingWindowAttentionParser() + args = ParsedArgs() + args.num_hidden_layers = 32 + result = parser.parse(args, vllm_config) + + assert result.sliding_window == 4096 + assert result.num_swa_layers == 32 + + +def test_sliding_window_parser_explicit_layer_types(): + """Test SlidingWindowAttentionParser with explicit layer_types list.""" + hf_config = Qwen3Config( + hidden_size=2048, + num_attention_heads=16, + num_hidden_layers=6, + ) + hf_config.sliding_window = 2048 + hf_config.layer_types = [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + ] + vllm_config = create_mock_vllm_config(hf_config) + parser = SlidingWindowAttentionParser() + args = ParsedArgs() + args.num_hidden_layers = 6 + result = parser.parse(args, vllm_config) + + assert result.sliding_window == 2048 + assert result.num_swa_layers == 3 + + +def test_sliding_window_parser_gemma2(): + """Test SlidingWindowAttentionParser detects alternating SWA for Gemma 2.""" + hf_config = Gemma2Config( + hidden_size=2304, + num_attention_heads=8, + num_key_value_heads=4, + num_hidden_layers=26, + sliding_window=4096, + ) + vllm_config = create_mock_vllm_config(hf_config) + parser = SlidingWindowAttentionParser() + args = ParsedArgs() + args.num_hidden_layers = 26 + result = parser.parse(args, vllm_config) + + assert result.sliding_window == 4096 + # Gemma 2 alternates: even layers are sliding window (26 layers -> 13 SWA) + assert result.num_swa_layers == 13 + + +def test_sliding_window_parser_disabled(): + """Test SlidingWindowAttentionParser respects disable_sliding_window.""" + hf_config = MistralConfig( + hidden_size=4096, + num_attention_heads=32, + num_key_value_heads=8, + num_hidden_layers=32, + sliding_window=4096, + ) + vllm_config = create_mock_vllm_config(hf_config, disable_sliding_window=True) + parser = SlidingWindowAttentionParser() + args = ParsedArgs() + args.num_hidden_layers = 32 + result = parser.parse(args, vllm_config) + + assert result.sliding_window is None + assert result.num_swa_layers == 0 + + +def test_attention_metrics_sliding_window_flops_bounds(): + """Test that SWA clamps attention core FLOPs when context_len > sliding_window.""" + sw = 1024 + dense_config = MistralConfig( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=8, + sliding_window=None, + ) + swa_config = MistralConfig( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=8, + sliding_window=sw, + ) + + dense_vllm = create_mock_vllm_config(dense_config) + swa_vllm = create_mock_vllm_config(swa_config) + + dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm) + swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm) + + # 1. When context <= sliding_window: FLOPs should be identical + short_ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=512, is_prefill=False + ) + assert swa_metrics.get_num_flops(short_ctx) == dense_metrics.get_num_flops( + short_ctx + ) + + # 2. When context > sliding_window: FLOPs for SWA must be strictly smaller + long_ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=8192, is_prefill=False + ) + dense_flops = dense_metrics.get_num_flops_breakdown(long_ctx) + swa_flops = swa_metrics.get_num_flops_breakdown(long_ctx) + + # Projections (qkv_proj, out_proj) are unaffected by sliding window + assert swa_flops["qkv_proj"] == dense_flops["qkv_proj"] + assert swa_flops["out_proj"] == dense_flops["out_proj"] + + # Attention core FLOPs are clamped: 8192 vs 1024 (8x difference) + assert swa_flops["attn_qk"] < dense_flops["attn_qk"] + assert swa_flops["attn_av"] < dense_flops["attn_av"] + assert swa_flops["attn_qk"] * 8 == dense_flops["attn_qk"] + assert swa_flops["attn_av"] * 8 == dense_flops["attn_av"] + + +def test_attention_metrics_sliding_window_read_bytes(): + """Test that SWA bounds decode KV-cache read bytes to sliding_window.""" + sw = 2048 + dense_config = MistralConfig( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=4, + num_hidden_layers=4, + sliding_window=None, + ) + swa_config = MistralConfig( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=4, + num_hidden_layers=4, + sliding_window=sw, + ) + + dense_vllm = create_mock_vllm_config(dense_config) + swa_vllm = create_mock_vllm_config(swa_config) + + dense_metrics = AttentionMetrics.from_vllm_config(dense_vllm) + swa_metrics = AttentionMetrics.from_vllm_config(swa_vllm) + + # Long decode context: 8192 tokens + ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=8192, is_prefill=False + ) + dense_reads = dense_metrics.get_read_bytes_breakdown(ctx) + swa_reads = swa_metrics.get_read_bytes_breakdown(ctx) + + # Attention input read traffic is lower for SWA + assert swa_reads["attn_input"] < dense_reads["attn_input"] + + # Weight reads and projection inputs are identical + assert swa_reads["qkv_weight"] == dense_reads["qkv_weight"] + assert swa_reads["out_weight"] == dense_reads["out_weight"] + assert swa_reads["qkv_input"] == dense_reads["qkv_input"] + + +def test_attention_metrics_hybrid_alternating_layers(): + """Test hybrid model where half layers are full attention and half are SWA.""" + sw = 1024 + hybrid_config = Qwen3Config( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=4, + ) + hybrid_config.sliding_window = sw + hybrid_config.layer_types = [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + ] + + vllm_config = create_mock_vllm_config(hybrid_config) + metrics = AttentionMetrics.from_vllm_config(vllm_config) + + assert metrics.num_swa_layers == 2 + assert metrics.sliding_window == sw + + ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=4096, is_prefill=False + ) + flops = metrics.get_num_flops_breakdown(ctx) + + # 2 full layers attend to 4096; 2 SWA layers attend to 1024 + # Expected TC total = 2 * 4096 + 2 * 1024 = 8192 + 2048 = 10240 + # Formula: 2 * q * d * (L_full * 4096 + L_swa * 1024) + q = metrics.num_attention_heads + d = metrics.head_dim + expected_core_flops = 2 * q * d * (2 * 4096 + 2 * 1024) + assert flops["attn_qk"] == expected_core_flops + assert flops["attn_av"] == expected_core_flops + + +def test_attention_metrics_sliding_window_per_gpu(): + """Test that SWA metrics scale properly with tensor and pipeline parallelism.""" + sw = 1024 + hf_config = MistralConfig( + hidden_size=4096, + num_attention_heads=32, + num_key_value_heads=8, + num_hidden_layers=16, + sliding_window=sw, + ) + single_vllm = create_mock_vllm_config(hf_config) + tp_vllm = create_mock_vllm_config(hf_config, tensor_parallel_size=2) + pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2) + + single_metrics = AttentionMetrics.from_vllm_config(single_vllm) + tp_metrics = AttentionMetrics.from_vllm_config(tp_vllm) + pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm) + + ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=4096, is_prefill=False + ) + + # Both TP=2 and PP=2 should halve the per-GPU attention core FLOPs + single_flops = single_metrics.get_num_flops_breakdown(ctx, per_gpu=True) + tp_flops = tp_metrics.get_num_flops_breakdown(ctx, per_gpu=True) + pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True) + + assert tp_flops["attn_qk"] == single_flops["attn_qk"] // 2 + assert pp_flops["attn_qk"] == single_flops["attn_qk"] // 2 + + +def test_attention_metrics_hybrid_sliding_window_per_gpu(): + """Test that hybrid SWA models preserve L_full + L_swa == L under PP.""" + sw = 1024 + hybrid_config = Gemma2Config( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=30, + sliding_window=sw, + ) + pp_vllm = create_mock_vllm_config(hybrid_config, pipeline_parallel_size=2) + pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm) + + ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=4096, is_prefill=False + ) + pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True) + pp_read_bytes = pp_metrics.get_read_bytes_breakdown(ctx, per_gpu=True) + + # Under PP=2, each global count is halved: 30 layers -> 15 per GPU, and the + # 15 SWA / 15 full split becomes 7.5 each. Gemma 2 alternates, so an even + # split is the correct average stage; deriving it after reducing L instead + # produced a lopsided 8/7. + q = pp_metrics.num_attention_heads + d = pp_metrics.head_dim + expected_pp_attn = int(2 * q * d * (7.5 * 4096 + 7.5 * sw)) + assert pp_flops["attn_qk"] == expected_pp_attn + assert pp_flops["attn_av"] == expected_pp_attn + assert "attn_input" in pp_read_bytes + + +def test_attention_metrics_minority_swa_survives_pipeline_parallelism(): + """A minority SWA sublayer must not round away under PP. + + Regression test for a 6-layer model with a single sliding-window layer at + pp_size=2: deriving the split after reducing L gave + round(3 * 1 / 6) == 0, billing all three per-GPU layers as full attention + and dropping the sliding-window contribution entirely. + """ + sw = 512 + context_len = 4096 + hf_config = Qwen3Config( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=6, + ) + hf_config.sliding_window = sw + hf_config.layer_types = ["sliding_attention"] + ["full_attention"] * 5 + + pp_vllm = create_mock_vllm_config(hf_config, pipeline_parallel_size=2) + pp_metrics = AttentionMetrics.from_vllm_config(pp_vllm) + assert pp_metrics.num_swa_layers == 1 + + ctx = ExecutionContext.from_single_request( + num_tokens=1, context_len=context_len, is_prefill=False + ) + pp_flops = pp_metrics.get_num_flops_breakdown(ctx, per_gpu=True) + + # 5 full and 1 SWA layer, each halved by PP=2: 2.5 full and 0.5 SWA. + q = pp_metrics.num_attention_heads + d = pp_metrics.head_dim + expected = int(2 * q * d * (2.5 * context_len + 0.5 * sw)) + assert pp_flops["attn_qk"] == expected + assert pp_flops["attn_av"] == expected + + # The sliding-window layer must change the result, i.e. it is not being + # billed at the full context length. + all_full = int(2 * q * d * 3 * context_len) + assert pp_flops["attn_qk"] < all_full + + +def test_attention_metrics_per_gpu_layers_sum_to_whole_model(): + """Per-GPU counts times pp_size must account for every layer. + + Flooring the layer count per stage silently drops the remainder, so a model + whose layers do not divide evenly across stages under-reports its FLOPs. + """ + hf_config = Qwen3Config( + hidden_size=2048, + num_attention_heads=16, + num_key_value_heads=16, + num_hidden_layers=7, + ) + pp_size = 2 + vllm_config = create_mock_vllm_config(hf_config, pipeline_parallel_size=pp_size) + metrics = AttentionMetrics.from_vllm_config(vllm_config) + + ctx = ExecutionContext.from_single_request( + num_tokens=8, context_len=1024, is_prefill=True + ) + per_gpu = metrics.get_num_flops_breakdown(ctx, per_gpu=True) + whole = metrics.get_num_flops_breakdown(ctx, per_gpu=False) + + for key, value in whole.items(): + assert per_gpu[key] * pp_size == pytest.approx(value, rel=1e-9) diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index 75524ef128..ab90ffcbe1 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -10,7 +10,7 @@ import json import time from abc import ABC, abstractmethod from collections.abc import Iterable -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from typing import Any, Protocol import prometheus_client @@ -124,8 +124,12 @@ class ExecutionContext: decode_context_len: int = 0 # sum of context_len for decode requests decode_token_context_product: int = 0 # sum of (num_tokens * context_len) + # Per-request records: (num_tokens, context_len, is_prefill) + requests: list[tuple[int, int, bool]] = field(default_factory=list) + def add(self, num_tokens: int, context_len: int, is_prefill: bool) -> None: """Add a single request's statistics to this batch context.""" + self.requests.append((num_tokens, context_len, is_prefill)) if is_prefill: self.num_prefill_requests += 1 self.prefill_num_tokens += num_tokens @@ -141,9 +145,40 @@ class ExecutionContext: """Total number of tokens across all requests in the batch.""" return self.prefill_num_tokens + self.decode_num_tokens - def total_token_context_product(self) -> int: - """Total sum of (num_tokens * context_len) across all requests.""" - return self.prefill_token_context_product + self.decode_token_context_product + def total_token_context_product(self, sliding_window: int | None = None) -> int: + """Total sum of (num_tokens * context_len) across all requests. + + If sliding_window is provided, context length per request is clamped to + sliding_window. + """ + if sliding_window is None: + return ( + self.prefill_token_context_product + self.decode_token_context_product + ) + return sum( + num_tokens * min(context_len, sliding_window) + for num_tokens, context_len, _ in self.requests + ) + + def prefill_context_len_for_window(self, sliding_window: int | None = None) -> int: + """Sum of context_len for prefill requests, clamped to sliding_window if set.""" + if sliding_window is None: + return self.prefill_context_len + return sum( + min(context_len, sliding_window) + for _, context_len, is_prefill in self.requests + if is_prefill + ) + + def decode_context_len_for_window(self, sliding_window: int | None = None) -> int: + """Sum of context_len for decode requests, clamped to sliding_window if set.""" + if sliding_window is None: + return self.decode_context_len + return sum( + min(context_len, sliding_window) + for _, context_len, is_prefill in self.requests + if not is_prefill + ) def num_logits_tokens(self) -> int: """Number of tokens that require logits computation (unembedding). @@ -214,6 +249,7 @@ class ParserChain: self.parsers.append(parser) def parse(self, vllm_config: VllmConfig) -> ParsedArgs: + """Apply all parsers in sequence and return the accumulated ParsedArgs.""" args = ParsedArgs() for parser in self.parsers: args = parser.parse(args, vllm_config) @@ -234,7 +270,9 @@ class ComponentMetrics(BaseModel, ABC): @classmethod @abstractmethod - def component_type(cls) -> str: ... + def component_type(cls) -> str: + """Return the unique string key identifying this component in the registry.""" + ... @classmethod @abstractmethod @@ -273,17 +311,23 @@ class ComponentMetrics(BaseModel, ABC): @abstractmethod def get_num_flops_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True - ) -> dict[str, int]: ... + ) -> dict[str, int]: + """Return per-operation FLOPs breakdown for this component.""" + ... @abstractmethod def get_read_bytes_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True - ) -> dict[str, int]: ... + ) -> dict[str, int]: + """Return per-operation read-memory-traffic breakdown for this component.""" + ... @abstractmethod def get_write_bytes_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True - ) -> dict[str, int]: ... + ) -> dict[str, int]: + """Return per-operation write-memory-traffic breakdown for this component.""" + ... def get_num_flops(self, ctx: ExecutionContext, per_gpu: bool = True) -> int: return sum(self.get_num_flops_breakdown(ctx, per_gpu).values()) @@ -306,6 +350,7 @@ class BaseConfigParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Parse base model dimensions, dtype byte sizes, and parallelism config.""" model_config = vllm_config.model_config args.vocab_size = model_config.get_vocab_size() @@ -357,6 +402,7 @@ class BaseAttentionConfigParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Parse KV head count, head dimension, and KV-cache dtype byte size.""" model_config = vllm_config.model_config args.num_key_value_heads = model_config.get_total_num_kv_heads() @@ -371,6 +417,54 @@ class BaseAttentionConfigParser(Parser): return args +class SlidingWindowAttentionParser(Parser): + """ + Parses sliding window attention configuration and layer breakdown. + Provides: sliding_window, num_swa_layers + """ + + def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Determine sliding_window size and count of SWA layers. + + Detects per-layer type lists (e.g. Llama-4), Gemma 2 alternating + pattern, and uniform SWA models (Mistral, Qwen2.5). + """ + model_config = vllm_config.model_config + + sliding_window: int | None = None + if not getattr(model_config, "disable_sliding_window", False): + sliding_window = model_config.get_sliding_window() + if sliding_window is None and hasattr(vllm_config, "cache_config"): + sliding_window = getattr( + vllm_config.cache_config, "sliding_window", None + ) + + args.sliding_window = sliding_window + + num_layers = args.num_hidden_layers + if sliding_window is not None and sliding_window > 0: + hf_text_config = getattr(model_config, "hf_text_config", None) + layer_types = getattr(hf_text_config, "layer_types", None) + if layer_types is not None and isinstance(layer_types, list): + num_swa = sum( + 1 for lt in layer_types[:num_layers] if "sliding" in lt.lower() + ) + args.num_swa_layers = num_swa + elif ( + getattr(getattr(model_config, "hf_config", None), "model_type", None) + == "gemma2" + ): + # Gemma 2 alternates sliding window: even layers (0, 2, ...) are sliding + args.num_swa_layers = (num_layers + 1) // 2 + else: + # Full SWA model (e.g. Mistral, Qwen2.5 with sliding_window) + args.num_swa_layers = num_layers + else: + args.num_swa_layers = 0 + + return args + + class AttentionQuantizationConfigParser(Parser): """ Parses quantization configuration for attention layers. @@ -378,6 +472,7 @@ class AttentionQuantizationConfigParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Override weight_byte_size based on the active quantization method.""" cfg = vllm_config.quant_config if cfg is None: @@ -401,6 +496,7 @@ class AttentionDetectionParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Raise InvalidComponent for MLA models so AttentionMetrics is skipped.""" if vllm_config.model_config.is_deepseek_mla: raise InvalidComponent( "Model uses MLA attention; use MLAAttentionMetrics instead" @@ -425,53 +521,129 @@ class AttentionMetrics(ComponentMetrics): # From BaseConfig Parser, overridden by AttentionQuantizationConfigParser weight_byte_size: int | float = Field(..., gt=0) - # TODO: discern cases where we have mixture of different attention layer types - # such as SWA, MLA, etc. + # From SlidingWindowAttentionParser + sliding_window: int | None = Field(None) + num_swa_layers: int = Field(0, ge=0) @classmethod def component_type(cls) -> str: + """Return the component registry key for standard (non-MLA) attention.""" return "attn" @classmethod def get_parser(cls) -> ParserChain: + """Return the parser chain for AttentionMetrics. + + Parsers run in order: MLA detection guard, base model config, + attention-specific config, sliding-window breakdown, quantization + weight-size override. + """ return ParserChain( AttentionDetectionParser(), BaseConfigParser(), BaseAttentionConfigParser(), + SlidingWindowAttentionParser(), AttentionQuantizationConfigParser(), ) + def _layer_counts(self, per_gpu: bool) -> tuple[float, float, float]: + """Return ``(L, L_full, L_swa)``, the layer counts to bill for. + + Per-GPU counts divide each global count by ``pp_size`` independently. + Deriving the sublayer split after reducing ``L`` instead rounds a + minority sublayer away: a 6-layer model with 1 sliding-window layer at + ``pp_size=2`` yields ``round(3 * 1 / 6) == 0``, billing all three layers + as full attention. Dividing each count separately keeps + ``L_full + L_swa == L`` and keeps the per-GPU values summing back to the + whole model across stages. + + The counts are fractional because these metrics are built from config + alone (see ``from_vllm_config``); no pipeline rank is available, so a + per-GPU figure is the average stage rather than a specific one. A model + whose layers do not divide evenly across stages has no single integer + answer here. + """ + L = float(self.num_hidden_layers) + L_swa = float(self.num_swa_layers) + L_full = L - L_swa + + if per_gpu: + L /= self.pp_size + L_swa /= self.pp_size + L_full /= self.pp_size + + return L, L_full, L_swa + def get_num_flops_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True ) -> dict[str, int]: - L, D, q, kv, d = ( - self.num_hidden_layers, + """Compute FLOPs breakdown for attention layers. + + Accounts for hybrid models by splitting total layers into full-attention + (L_full) and sliding-window (L_swa) sublayers. See ``_layer_counts`` for + how those are scaled under pipeline parallelism. + + Args: + ctx: Execution context describing the current batch. + per_gpu: If True, scale counts down by tensor/pipeline parallelism. + + Returns: + Dict with keys: qkv_proj, out_proj, attn_qk, attn_av. + """ + D, q, kv, d = ( self.hidden_size, self.num_attention_heads, self.num_key_value_heads, self.head_dim, ) T = ctx.total_num_tokens() - TC = ctx.total_token_context_product() + TC_full = ctx.total_token_context_product() + + L, L_full, L_swa = self._layer_counts(per_gpu) if per_gpu: - L //= self.pp_size # tensor parallel along heads q = max(1, q // self.tp_size) kv = max(1, kv // self.tp_size) - return { - "qkv_proj": 2 * T * D * (q + 2 * kv) * d * L, - "attn_qk": 2 * q * TC * d * L, - "attn_av": 2 * q * TC * d * L, - "out_proj": 2 * T * D * q * d * L, + flops = { + "qkv_proj": int(2 * T * D * (q + 2 * kv) * d * L), + "out_proj": int(2 * T * D * q * d * L), } + attn_qk = 0.0 + attn_av = 0.0 + if L_full > 0: + attn_qk += 2 * q * TC_full * d * L_full + attn_av += 2 * q * TC_full * d * L_full + + if L_swa > 0 and self.sliding_window is not None: + TC_swa = ctx.total_token_context_product(self.sliding_window) + attn_qk += 2 * q * TC_swa * d * L_swa + attn_av += 2 * q * TC_swa * d * L_swa + + flops["attn_qk"] = int(attn_qk) + flops["attn_av"] = int(attn_av) + + return flops + def get_read_bytes_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True ) -> dict[str, int]: - L, D, q, kv, d = ( - self.num_hidden_layers, + """Compute read-memory-traffic breakdown for attention layers. + + Splits layers into full-attention and sliding-window sublayers using the + same counts as get_num_flops_breakdown; see ``_layer_counts``. + + Args: + ctx: Execution context describing the current batch. + per_gpu: If True, scale counts down by tensor/pipeline parallelism. + + Returns: + Dict with keys: qkv_input, qkv_weight, attn_input (conditional), + out_input, out_weight. + """ + D, q, kv, d = ( self.hidden_size, self.num_attention_heads, self.num_key_value_heads, @@ -479,35 +651,63 @@ class AttentionMetrics(ComponentMetrics): ) T = ctx.total_num_tokens() + L, L_full, L_swa = self._layer_counts(per_gpu) + if per_gpu: - L //= self.pp_size # tensor parallel along heads q = max(1, q // self.tp_size) kv = max(1, kv // self.tp_size) read_bytes = {} - read_bytes["qkv_input"] = T * D * self.activation_byte_size * L + read_bytes["qkv_input"] = int(T * D * self.activation_byte_size * L) read_bytes["qkv_weight"] = int(D * (q + 2 * kv) * d * self.weight_byte_size * L) - # Attention input reads differ between prefill and decode - # Prefill: read Q, K, V activations (all in activation_byte_size) - if ctx.prefill_num_tokens > 0: - read_bytes["attn_input"] = ( - (ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv) - * d - * self.activation_byte_size - * L - ) + attn_input = 0.0 - # Decode: read Q activations + read K, V from cache (in cache_byte_size) - if ctx.decode_num_tokens > 0: - read_bytes["attn_input"] = read_bytes.get("attn_input", 0) + ( - ctx.decode_num_tokens * q * d * self.activation_byte_size * L - + 2 * ctx.decode_context_len * kv * d * self.cache_byte_size * L - ) + # Full attention layers + if L_full > 0: + if ctx.prefill_num_tokens > 0: + attn_input += ( + (ctx.prefill_num_tokens * q + 2 * ctx.prefill_context_len * kv) + * d + * self.activation_byte_size + * L_full + ) + if ctx.decode_num_tokens > 0: + attn_input += ( + ctx.decode_num_tokens * q * d * self.activation_byte_size * L_full + + 2 + * ctx.decode_context_len + * kv + * d + * self.cache_byte_size + * L_full + ) - read_bytes["out_input"] = T * q * d * self.activation_byte_size * L + # Sliding window attention layers + if L_swa > 0: + sw = self.sliding_window + prefill_sw_len = ctx.prefill_context_len_for_window(sw) + decode_sw_len = ctx.decode_context_len_for_window(sw) + + if ctx.prefill_num_tokens > 0: + attn_input += ( + (ctx.prefill_num_tokens * q + 2 * prefill_sw_len * kv) + * d + * self.activation_byte_size + * L_swa + ) + if ctx.decode_num_tokens > 0: + attn_input += ( + ctx.decode_num_tokens * q * d * self.activation_byte_size * L_swa + + 2 * decode_sw_len * kv * d * self.cache_byte_size * L_swa + ) + + if ctx.prefill_num_tokens > 0 or ctx.decode_num_tokens > 0: + read_bytes["attn_input"] = int(attn_input) + + read_bytes["out_input"] = int(T * q * d * self.activation_byte_size * L) read_bytes["out_weight"] = int(q * d * D * self.weight_byte_size * L) return read_bytes @@ -515,9 +715,13 @@ class AttentionMetrics(ComponentMetrics): def get_write_bytes_breakdown( self, ctx: ExecutionContext, per_gpu: bool = True ) -> dict[str, int]: - """Calculate write memory traffic for attention layers.""" - L, D, q, kv, d = ( - self.num_hidden_layers, + """Calculate write memory traffic for attention layers. + + Writes do not depend on the attention window, so only the total layer + count is needed; it comes from ``_layer_counts`` so all three breakdowns + bill the same number of layers. + """ + D, q, kv, d = ( self.hidden_size, self.num_attention_heads, self.num_key_value_heads, @@ -525,16 +729,17 @@ class AttentionMetrics(ComponentMetrics): ) T = ctx.total_num_tokens() + L, _, _ = self._layer_counts(per_gpu) + if per_gpu: - L //= self.pp_size # tensor parallel along heads q = max(1, q // self.tp_size) kv = max(1, kv // self.tp_size) return { - "qkv_output": T * (q + 2 * kv) * d * self.activation_byte_size * L, - "kv_cache": 2 * T * kv * d * self.cache_byte_size * L, - "out_output": T * D * self.activation_byte_size * L, + "qkv_output": int(T * (q + 2 * kv) * d * self.activation_byte_size * L), + "kv_cache": int(2 * T * kv * d * self.cache_byte_size * L), + "out_output": int(T * D * self.activation_byte_size * L), } @@ -549,6 +754,7 @@ class MLADetectionParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Raise InvalidComponent if the model does not use MLA attention.""" if not vllm_config.model_config.is_deepseek_mla: raise InvalidComponent("Model does not use MLA attention") return args @@ -562,6 +768,7 @@ class MLAConfigParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Parse MLA-specific compression dimensions and KV-cache dtype byte size.""" model_config = vllm_config.model_config cfg = model_config.hf_text_config @@ -616,10 +823,16 @@ class MLAAttentionMetrics(ComponentMetrics): @classmethod def component_type(cls) -> str: + """Return the component registry key for MLA attention.""" return "mla_attn" @classmethod def get_parser(cls) -> ParserChain: + """Return the parser chain for MLAAttentionMetrics. + + Parsers run in order: MLA detection guard, base model config, + MLA-specific config, quantization weight-size override. + """ return ParserChain( MLADetectionParser(), BaseConfigParser(), @@ -923,6 +1136,7 @@ class FfnQuantizationConfigParser(Parser): """ def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: + """Override weight_byte_size based on the active FFN quantization method.""" cfg = vllm_config.quant_config if cfg is None: @@ -980,10 +1194,16 @@ class FfnMetrics(ComponentMetrics): @classmethod def component_type(cls) -> str: + """Return the component registry key for FFN layers.""" return "ffn" @classmethod def get_parser(cls) -> ParserChain: + """Return the parser chain for FfnMetrics. + + Parsers run in order: base model config, FFN parallelism, base FFN + config, MoE interleave step, MoE layer frequency, quantization override. + """ return ParserChain( BaseConfigParser(), FfnParallelParser(), @@ -1207,10 +1427,12 @@ class UnembedMetrics(ComponentMetrics): @classmethod def component_type(cls) -> str: + """Return the component registry key for the unembedding (LM head) layer.""" return "unembed" @classmethod def get_parser(cls) -> ParserChain: + """Return the parser chain for UnembedMetrics (base model config only).""" return ParserChain( BaseConfigParser(), )