[Bugfix] Fix degenerate KV cache stride causing TMA cudaErrorIllegalInstruction (#40737)

Signed-off-by: David Oy <david@baseten.co>
Signed-off-by: David Oy <58150256+the-david-oy@users.noreply.github.com>
Signed-off-by: David Oy <david.oy@baseten.co>
Co-authored-by: David Oy <david@baseten.co>
Co-authored-by: Claude <claude@anthropic.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
David Oy
2026-05-03 23:52:18 +00:00
committed by GitHub
co-authored by David Oy Claude Vadim Gimpelson
parent db9a84e0cd
commit 66dfee7121
6 changed files with 267 additions and 22 deletions
+1 -1
View File
@@ -125,7 +125,7 @@ def get_allowed_cpu_list() -> list[LogicalCPUInfo]:
if platform.system() == "Darwin":
return cpu_list
global_allowed_cpu_id_list = os.sched_getaffinity(0)
global_allowed_cpu_id_list = os.sched_getaffinity(0) # type: ignore[attr-defined]
logical_cpu_list = [x for x in cpu_list if x.id in global_allowed_cpu_id_list]
return logical_cpu_list
+26
View File
@@ -110,6 +110,32 @@ def is_strictly_contiguous(t: torch.Tensor) -> bool:
return True
def canonicalize_singleton_dim_strides(t: torch.Tensor) -> torch.Tensor:
"""Fix degenerate strides on size=1 dimensions for CUDA TMA compatibility.
PyTorch allows any stride on a size=1 dim (is_contiguous() is always True
there), so a size=1 dim may have stride=1 (2 bytes for bf16) instead of
the canonical product(shape[i+1:]). CUDA TMA on H100+ requires all
non-outermost strides to be ≥16-byte aligned; stride=1 triggers
cudaErrorIllegalInstruction. Zero-copy: patches stride metadata only via
as_strided; returns t unchanged if all size=1 strides are already canonical.
"""
if 1 not in t.shape:
return t
strides = list(t.stride())
shape = t.shape
prev_stride = 1
changed = False
for i in range(len(shape) - 1, -1, -1):
if shape[i] == 1 and strides[i] != prev_stride:
strides[i] = prev_stride
changed = True
prev_stride = strides[i] * shape[i]
if not changed:
return t
return t.as_strided(t.shape, strides)
@contextlib.contextmanager
def set_default_torch_dtype(dtype: torch.dtype):
"""Sets the default torch dtype to the given dtype."""