1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
16
17# pylint: disable=no-name-in-module,consider-using-from-import,pointless-string-statement
18# pylint: disable=redefined-outer-name,missing-public-type-hints,missing-public-docstring
19# pylint: disable=unused-argument,no-member,import-outside-toplevel
20# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring
21# pylint: disable=missing-class-docstring,broad-exception-caught,protected-access
22
23import itertools
24import contextlib
25import os
26import functools
27import warnings
28import logging
29from enum import Enum
30from functools import lru_cache
31from typing import Any, Callable, Optional
32from packaging import version
33
34import torch
35import triton
36import triton.language as tl
37import triton.language.extra.libdevice as tldevice
38import triton.runtime.driver as driver
39
40logger = logging.getLogger(__name__)
41
42FLA_CI_ENV = os.getenv("FLA_CI_ENV") == "1"
43
44
45def tensor_cache(fn: Optional[Callable[..., torch.Tensor]] = None, *, maxsize: int = 1) -> Any:
46 """
47 A decorator that caches the most recent results of a function with tensor inputs.
48
49 This decorator will store the outputs of the decorated function for the most recent
50 set of input tensors, up to `maxsize` entries. If the function is called again with
51 the same input tensors, it will return the cached result.
52
53 When maxsize=1 (default), the behavior is identical to caching only the most recent result.
54 Can be used as @tensor_cache or @tensor_cache(maxsize=n).
55
56 Args:
57 fn (Callable[..., torch.Tensor], optional):
58 The function to be decorated when used without parentheses.
59 maxsize (int):
60 Maximum number of input combinations to cache. Default is 1.
61
62 Returns:
63 Callable[..., torch.Tensor]:
64 A wrapped version of the input function with caching.
65 """
66 if maxsize < 1:
67 raise ValueError("maxsize must be at least 1")
68
69 def _is_match(a: Any, b: Any) -> bool:
70 if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
71 return a is b
72 try:
73 return a == b
74 except Exception:
75 return a is b
76
77 def _make_wrapper(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]:
78 cache: list = []
79
80 @functools.wraps(fn)
81 def wrapper(*args: Any, **kwargs: Any) -> Any:
82 for i, (cached_args, cached_kwargs, cached_result) in enumerate(cache):
83 if len(args) == len(cached_args) and len(kwargs) == len(cached_kwargs):
84 if all(_is_match(a, b) for a, b in zip(args, cached_args)) and all(
85 k in cached_kwargs and _is_match(v, cached_kwargs[k]) for k, v in kwargs.items()
86 ):
87 if i != 0:
88 cache.insert(0, cache.pop(i))
89 return cached_result
90
91 result = fn(*args, **kwargs)
92 cache.insert(0, (args, kwargs, result))
93 if len(cache) > maxsize:
94 cache.pop()
95 return result
96
97 return wrapper
98
99 if fn is not None:
100 return _make_wrapper(fn)
101 return _make_wrapper
102
103
104@tensor_cache
105def prepare_lens(cu_seqlens: torch.LongTensor) -> torch.LongTensor:
106 return cu_seqlens[1:] - cu_seqlens[:-1]
107
108
109@tensor_cache(maxsize=3)
110def prepare_chunk_indices(cu_seqlens: torch.LongTensor, chunk_size: int) -> torch.LongTensor:
111 indices = torch.cat([torch.arange(n) for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()])
112 return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens)
113
114
115def get_abs_err(x, y):
116 return (x.detach() - y.detach()).flatten().abs().max().item()
117
118
119def get_err_ratio(x, y):
120 err = (x.detach() - y.detach()).flatten().square().mean().sqrt().item()
121 base = (x.detach()).flatten().square().mean().sqrt().item()
122 return err / (base + 1e-8)
123
124
125def assert_close(prefix, ref, tri, ratio, warning=False, err_atol=1e-6):
126 abs_atol = get_abs_err(ref, tri)
127 msg = f"{prefix:>16} diff: {abs_atol:.6f} ratio: {get_err_ratio(ref, tri):.6f}"
128 logger.info(msg)
129 error_rate = get_err_ratio(ref, tri)
130 if abs_atol <= err_atol:
131 return
132 allow_warning = warning or (FLA_CI_ENV and (error_rate < 0.01 or abs_atol <= 0.3))
133 if allow_warning:
134 if error_rate > ratio:
135 warnings.warn(msg)
136 else:
137 if not error_rate < ratio:
138 raise AssertionError(msg)
139
140
141if hasattr(triton.language, '_experimental_make_tensor_descriptor'):
142 # For Triton 3.3.x
143 make_tensor_descriptor = triton.language._experimental_make_tensor_descriptor
144elif hasattr(triton.language, 'make_tensor_descriptor'):
145 # For Triton 3.4.x and later
146 make_tensor_descriptor = triton.language.make_tensor_descriptor
147else:
148 """
149 Fallback implementation when TMA is not supported.
150 Returns None to indicate TMA descriptors are unavailable.
151 Just make triton compiler happy.
152 """
153
154 @triton.jit
155 def make_tensor_descriptor(
156 base,
157 shape,
158 strides,
159 block_shape,
160 _builder=None,
161 ):
162 return None
163
164
165@lru_cache(maxsize=None)
166def get_available_device() -> str:
167 try:
168 return triton.runtime.driver.active.get_current_target().backend
169 except Exception:
170 _cpu_device_warning()
171 return 'cpu'
172
173
174def map_triton_backend_to_torch_device() -> str:
175 backend = get_available_device() # 'cuda' | 'hip' | 'xpu' | 'cpu' | ...
176 return {'cuda': 'cuda', 'hip': 'cuda', 'xpu': 'xpu'}.get(backend, backend)
177
178
179device = get_available_device() if get_available_device() != 'hip' else 'cuda'
180device_torch_lib = getattr(torch, device)
181device_platform = get_available_device()
182is_amd = device_platform == 'hip'
183is_nvidia = device_platform == 'cuda'
184is_nvidia_hopper = is_nvidia and (
185 'NVIDIA H' in torch.cuda.get_device_name(0) or torch.cuda.get_device_capability()[0] >= 9
186)
187
188is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8
189is_tma_supported = (
190 (is_nvidia and torch.cuda.get_device_capability(0)[0] >= 9)
191 and os.environ.get('FLA_NO_USE_TMA', '0') != '1'
192 and (
193 hasattr(triton.language, '_experimental_make_tensor_descriptor')
194 or hasattr(triton.language, 'make_tensor_descriptor')
195 )
196)
197
198if is_nvidia and not is_tf32_supported:
199 # Make old card happy, since triton will use tf32 by default.
200 # This is a workaround for old nvidia card.
201 os.environ['TRITON_F32_DEFAULT'] = 'ieee'
202
203
204@lru_cache(maxsize=None)
205def check_pytorch_version(version_s: str = '2.4') -> bool:
206 return version.parse(torch.__version__) >= version.parse(version_s)
207
208
209if check_pytorch_version('2.4'):
210 device = 'cuda' if device == 'cpu' else device
211 autocast_custom_fwd = functools.partial(torch.amp.custom_fwd, device_type=device)
212 autocast_custom_bwd = functools.partial(torch.amp.custom_bwd, device_type=device)
213
214 def custom_device_ctx(index: int):
215 return device_torch_lib.device(index)
216else:
217 if device != 'cuda':
218 raise AssertionError('Only cuda device is supported for PyTorch version < 2.4.0.')
219 autocast_custom_fwd = device_torch_lib.amp.custom_fwd
220 autocast_custom_bwd = device_torch_lib.amp.custom_bwd
221
222 def custom_device_ctx(index: int):
223 return torch.cuda.device(index)
224
225
226def input_guard(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]:
227 """
228 A decorator to make sure all input tensors are contiguous and set the device based on input tensors.
229 """
230
231 @functools.wraps(fn)
232 def wrapper(*args, **kwargs):
233 contiguous_args = (i if not isinstance(i, torch.Tensor) else i.contiguous() for i in args)
234 contiguous_kwargs = {k: (v if not isinstance(v, torch.Tensor) else v.contiguous()) for k, v in kwargs.items()}
235
236 tensor = None
237 for arg in args:
238 if isinstance(arg, torch.Tensor):
239 tensor = arg
240 break
241 if tensor is None:
242 for value in kwargs.values():
243 if isinstance(value, torch.Tensor):
244 tensor = value
245 break
246
247 if tensor is not None:
248 ctx = custom_device_ctx(tensor.device.index)
249 else:
250 ctx = contextlib.nullcontext()
251
252 with ctx:
253 return fn(*contiguous_args, **contiguous_kwargs)
254
255 return wrapper
256
257
258def _cpu_device_warning():
259 warnings.warn(('Triton is not supported on current platform, roll back to CPU.'), stacklevel=1)
260
261
262@tensor_cache
263def prepare_chunk_offsets(cu_seqlens: torch.LongTensor, chunk_size: int) -> torch.LongTensor:
264 return torch.cat([cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]).cumsum(-1)
265
266
267if os.environ.get('FLA_USE_FAST_OPS', '0') == '1':
268 exp = tldevice.fast_expf
269 exp2 = tldevice.exp2
270 log = tldevice.fast_logf
271 log2 = tldevice.fast_log2f
272else:
273 exp = tl.exp
274 exp2 = tl.math.exp2
275 log = tl.log
276 log2 = tl.log2
277
278
279def get_all_max_shared_mem():
280 try:
281 return [
282 triton.runtime.driver.active.utils.get_device_properties(i)['max_shared_mem']
283 for i in range(device_torch_lib.device_count())
284 ]
285 except Exception:
286 _cpu_device_warning()
287 return [-1]
288
289
290class Backend(Enum):
291 ADA = 101376 # RTX 4090
292 AMPERE = 166912 # A100
293 HOPPER = 232448 # H100
294 DEFAULT = 102400 # Default
295
296 @classmethod
297 def get_shared_memory(cls, arch: str) -> int:
298 try:
299 return cls[arch.upper()].value
300 except KeyError:
301 return cls.DEFAULT.value
302
303
304@lru_cache(maxsize=None)
305def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:
306 try:
307 device_shared_mem_list = get_all_max_shared_mem()
308 max_shared_memory = device_shared_mem_list[tensor_idx]
309 return max_shared_memory >= Backend.get_shared_memory(arch)
310 except Exception:
311 return False
312
313
314def get_autotune_config(
315 multibuffer_list: tuple = (False,),
316 unit_flag_list: tuple = (False,),
317 limit_auto_multi_buffer_only_for_local_buffer_list: tuple = (False,),
318 limit_auto_multi_buffer_of_local_buffer_list: tuple = ("no-l0c",),
319 set_workspace_multibuffer_list: tuple = (2, 4),
320 enable_hivm_auto_cv_balance_list: tuple = (True,),
321 tile_mix_vector_loop_num_list: tuple = (2, 4),
322 tile_mix_cube_loop_num_list: tuple = (2, 4),
323):
324 configs = []
325 for (
326 multibuffer,
327 unit_flag,
328 limit_auto_multi_buffer_only_for_local_buffer,
329 limit_auto_multi_buffer_of_local_buffer,
330 ) in itertools.product(
331 list(multibuffer_list),
332 list(unit_flag_list),
333 list(limit_auto_multi_buffer_only_for_local_buffer_list),
334 list(limit_auto_multi_buffer_of_local_buffer_list),
335 ):
336 base_config_dict = {
337 'multibuffer': multibuffer,
338 'unit_flag': unit_flag,
339 'limit_auto_multi_buffer_only_for_local_buffer': limit_auto_multi_buffer_only_for_local_buffer,
340 'limit_auto_multi_buffer_of_local_buffer': limit_auto_multi_buffer_of_local_buffer,
341 }
342
343 if limit_auto_multi_buffer_only_for_local_buffer:
344 configs.append(triton.Config(base_config_dict))
345 else:
346 for (
347 set_workspace_multibuffer,
348 enable_hivm_auto_cv_balance,
349 tile_mix_vector_loop,
350 tile_mix_cube_loop,
351 ) in itertools.product(
352 list(set_workspace_multibuffer_list),
353 list(enable_hivm_auto_cv_balance_list),
354 list(tile_mix_vector_loop_num_list),
355 list(tile_mix_cube_loop_num_list),
356 ):
357 full_config_dict = base_config_dict.copy()
358 full_config_dict.update(
359 {
360 'set_workspace_multibuffer': set_workspace_multibuffer,
361 'enable_hivm_auto_cv_balance': enable_hivm_auto_cv_balance,
362 'tile_mix_vector_loop': tile_mix_vector_loop,
363 'tile_mix_cube_loop': tile_mix_cube_loop,
364 }
365 )
366 configs.append(triton.Config(full_config_dict))
367 return configs
368
369
370def get_npu_properties():
371 return driver.active.utils.get_device_properties(torch.npu.current_device())
372
373
374@functools.cache
375def get_vector_num() -> int:
376 import torch_npu
377
378 current_device = torch_npu.npu.current_device()
379 properties = driver.active.utils.get_device_properties(current_device)
380 return properties["num_vectorcore"]
381
382
383@lru_cache
384def is_arch35():
385 try:
386 import torch_npu
387
388 return "Ascend910_95" in torch_npu.npu.get_device_name() or "Ascend950" in torch_npu.npu.get_device_name()
389 except Exception:
390 return False