[Bugfix] Fix degenerate KV cache stride causing TMA cudaErrorIllegalInstruction (#40737)
Signed-off-by: David Oy <david@baseten.co> Signed-off-by: David Oy <58150256+the-david-oy@users.noreply.github.com> Signed-off-by: David Oy <david.oy@baseten.co> Co-authored-by: David Oy <david@baseten.co> Co-authored-by: Claude <claude@anthropic.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
David Oy
Claude
Vadim Gimpelson
parent
db9a84e0cd
commit
66dfee7121
@@ -125,7 +125,7 @@ def get_allowed_cpu_list() -> list[LogicalCPUInfo]:
|
||||
if platform.system() == "Darwin":
|
||||
return cpu_list
|
||||
|
||||
global_allowed_cpu_id_list = os.sched_getaffinity(0)
|
||||
global_allowed_cpu_id_list = os.sched_getaffinity(0) # type: ignore[attr-defined]
|
||||
logical_cpu_list = [x for x in cpu_list if x.id in global_allowed_cpu_id_list]
|
||||
|
||||
return logical_cpu_list
|
||||
|
||||
@@ -110,6 +110,32 @@ def is_strictly_contiguous(t: torch.Tensor) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def canonicalize_singleton_dim_strides(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Fix degenerate strides on size=1 dimensions for CUDA TMA compatibility.
|
||||
|
||||
PyTorch allows any stride on a size=1 dim (is_contiguous() is always True
|
||||
there), so a size=1 dim may have stride=1 (2 bytes for bf16) instead of
|
||||
the canonical product(shape[i+1:]). CUDA TMA on H100+ requires all
|
||||
non-outermost strides to be ≥16-byte aligned; stride=1 triggers
|
||||
cudaErrorIllegalInstruction. Zero-copy: patches stride metadata only via
|
||||
as_strided; returns t unchanged if all size=1 strides are already canonical.
|
||||
"""
|
||||
if 1 not in t.shape:
|
||||
return t
|
||||
strides = list(t.stride())
|
||||
shape = t.shape
|
||||
prev_stride = 1
|
||||
changed = False
|
||||
for i in range(len(shape) - 1, -1, -1):
|
||||
if shape[i] == 1 and strides[i] != prev_stride:
|
||||
strides[i] = prev_stride
|
||||
changed = True
|
||||
prev_stride = strides[i] * shape[i]
|
||||
if not changed:
|
||||
return t
|
||||
return t.as_strided(t.shape, strides)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_default_torch_dtype(dtype: torch.dtype):
|
||||
"""Sets the default torch dtype to the given dtype."""
|
||||
|
||||
Reference in New Issue
Block a user