Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / _triton / gated_delta_net / utils.py: 0%

201 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16 

17# pylint: disable=no-name-in-module,consider-using-from-import,pointless-string-statement 

18# pylint: disable=redefined-outer-name,missing-public-type-hints,missing-public-docstring 

19# pylint: disable=unused-argument,no-member,import-outside-toplevel 

20# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring 

21# pylint: disable=missing-class-docstring,broad-exception-caught,protected-access 

22 

23import itertools 

24import contextlib 

25import os 

26import functools 

27import warnings 

28import logging 

29from enum import Enum 

30from functools import lru_cache 

31from typing import Any, Callable, Optional 

32from packaging import version 

33 

34import torch 

35import triton 

36import triton.language as tl 

37import triton.language.extra.libdevice as tldevice 

38import triton.runtime.driver as driver 

39 

40logger = logging.getLogger(__name__) 

41 

42FLA_CI_ENV = os.getenv("FLA_CI_ENV") == "1" 

43 

44 

45def tensor_cache(fn: Optional[Callable[..., torch.Tensor]] = None, *, maxsize: int = 1) -> Any: 

46 """ 

47 A decorator that caches the most recent results of a function with tensor inputs. 

48 

49 This decorator will store the outputs of the decorated function for the most recent 

50 set of input tensors, up to `maxsize` entries. If the function is called again with 

51 the same input tensors, it will return the cached result. 

52 

53 When maxsize=1 (default), the behavior is identical to caching only the most recent result. 

54 Can be used as @tensor_cache or @tensor_cache(maxsize=n). 

55 

56 Args: 

57 fn (Callable[..., torch.Tensor], optional): 

58 The function to be decorated when used without parentheses. 

59 maxsize (int): 

60 Maximum number of input combinations to cache. Default is 1. 

61 

62 Returns: 

63 Callable[..., torch.Tensor]: 

64 A wrapped version of the input function with caching. 

65 """ 

66 if maxsize < 1: 

67 raise ValueError("maxsize must be at least 1") 

68 

69 def _is_match(a: Any, b: Any) -> bool: 

70 if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor): 

71 return a is b 

72 try: 

73 return a == b 

74 except Exception: 

75 return a is b 

76 

77 def _make_wrapper(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: 

78 cache: list = [] 

79 

80 @functools.wraps(fn) 

81 def wrapper(*args: Any, **kwargs: Any) -> Any: 

82 for i, (cached_args, cached_kwargs, cached_result) in enumerate(cache): 

83 if len(args) == len(cached_args) and len(kwargs) == len(cached_kwargs): 

84 if all(_is_match(a, b) for a, b in zip(args, cached_args)) and all( 

85 k in cached_kwargs and _is_match(v, cached_kwargs[k]) for k, v in kwargs.items() 

86 ): 

87 if i != 0: 

88 cache.insert(0, cache.pop(i)) 

89 return cached_result 

90 

91 result = fn(*args, **kwargs) 

92 cache.insert(0, (args, kwargs, result)) 

93 if len(cache) > maxsize: 

94 cache.pop() 

95 return result 

96 

97 return wrapper 

98 

99 if fn is not None: 

100 return _make_wrapper(fn) 

101 return _make_wrapper 

102 

103 

104@tensor_cache 

105def prepare_lens(cu_seqlens: torch.LongTensor) -> torch.LongTensor: 

106 return cu_seqlens[1:] - cu_seqlens[:-1] 

107 

108 

109@tensor_cache(maxsize=3) 

110def prepare_chunk_indices(cu_seqlens: torch.LongTensor, chunk_size: int) -> torch.LongTensor: 

111 indices = torch.cat([torch.arange(n) for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()]) 

112 return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens) 

113 

114 

115def get_abs_err(x, y): 

116 return (x.detach() - y.detach()).flatten().abs().max().item() 

117 

118 

119def get_err_ratio(x, y): 

120 err = (x.detach() - y.detach()).flatten().square().mean().sqrt().item() 

121 base = (x.detach()).flatten().square().mean().sqrt().item() 

122 return err / (base + 1e-8) 

123 

124 

125def assert_close(prefix, ref, tri, ratio, warning=False, err_atol=1e-6): 

126 abs_atol = get_abs_err(ref, tri) 

127 msg = f"{prefix:>16} diff: {abs_atol:.6f} ratio: {get_err_ratio(ref, tri):.6f}" 

128 logger.info(msg) 

129 error_rate = get_err_ratio(ref, tri) 

130 if abs_atol <= err_atol: 

131 return 

132 allow_warning = warning or (FLA_CI_ENV and (error_rate < 0.01 or abs_atol <= 0.3)) 

133 if allow_warning: 

134 if error_rate > ratio: 

135 warnings.warn(msg) 

136 else: 

137 if not error_rate < ratio: 

138 raise AssertionError(msg) 

139 

140 

141if hasattr(triton.language, '_experimental_make_tensor_descriptor'): 

142 # For Triton 3.3.x 

143 make_tensor_descriptor = triton.language._experimental_make_tensor_descriptor 

144elif hasattr(triton.language, 'make_tensor_descriptor'): 

145 # For Triton 3.4.x and later 

146 make_tensor_descriptor = triton.language.make_tensor_descriptor 

147else: 

148 """ 

149 Fallback implementation when TMA is not supported. 

150 Returns None to indicate TMA descriptors are unavailable. 

151 Just make triton compiler happy. 

152 """ 

153 

154 @triton.jit 

155 def make_tensor_descriptor( 

156 base, 

157 shape, 

158 strides, 

159 block_shape, 

160 _builder=None, 

161 ): 

162 return None 

163 

164 

165@lru_cache(maxsize=None) 

166def get_available_device() -> str: 

167 try: 

168 return triton.runtime.driver.active.get_current_target().backend 

169 except Exception: 

170 _cpu_device_warning() 

171 return 'cpu' 

172 

173 

174def map_triton_backend_to_torch_device() -> str: 

175 backend = get_available_device() # 'cuda' | 'hip' | 'xpu' | 'cpu' | ... 

176 return {'cuda': 'cuda', 'hip': 'cuda', 'xpu': 'xpu'}.get(backend, backend) 

177 

178 

179device = get_available_device() if get_available_device() != 'hip' else 'cuda' 

180device_torch_lib = getattr(torch, device) 

181device_platform = get_available_device() 

182is_amd = device_platform == 'hip' 

183is_nvidia = device_platform == 'cuda' 

184is_nvidia_hopper = is_nvidia and ( 

185 'NVIDIA H' in torch.cuda.get_device_name(0) or torch.cuda.get_device_capability()[0] >= 9 

186) 

187 

188is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8 

189is_tma_supported = ( 

190 (is_nvidia and torch.cuda.get_device_capability(0)[0] >= 9) 

191 and os.environ.get('FLA_NO_USE_TMA', '0') != '1' 

192 and ( 

193 hasattr(triton.language, '_experimental_make_tensor_descriptor') 

194 or hasattr(triton.language, 'make_tensor_descriptor') 

195 ) 

196) 

197 

198if is_nvidia and not is_tf32_supported: 

199 # Make old card happy, since triton will use tf32 by default. 

200 # This is a workaround for old nvidia card. 

201 os.environ['TRITON_F32_DEFAULT'] = 'ieee' 

202 

203 

204@lru_cache(maxsize=None) 

205def check_pytorch_version(version_s: str = '2.4') -> bool: 

206 return version.parse(torch.__version__) >= version.parse(version_s) 

207 

208 

209if check_pytorch_version('2.4'): 

210 device = 'cuda' if device == 'cpu' else device 

211 autocast_custom_fwd = functools.partial(torch.amp.custom_fwd, device_type=device) 

212 autocast_custom_bwd = functools.partial(torch.amp.custom_bwd, device_type=device) 

213 

214 def custom_device_ctx(index: int): 

215 return device_torch_lib.device(index) 

216else: 

217 if device != 'cuda': 

218 raise AssertionError('Only cuda device is supported for PyTorch version < 2.4.0.') 

219 autocast_custom_fwd = device_torch_lib.amp.custom_fwd 

220 autocast_custom_bwd = device_torch_lib.amp.custom_bwd 

221 

222 def custom_device_ctx(index: int): 

223 return torch.cuda.device(index) 

224 

225 

226def input_guard(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: 

227 """ 

228 A decorator to make sure all input tensors are contiguous and set the device based on input tensors. 

229 """ 

230 

231 @functools.wraps(fn) 

232 def wrapper(*args, **kwargs): 

233 contiguous_args = (i if not isinstance(i, torch.Tensor) else i.contiguous() for i in args) 

234 contiguous_kwargs = {k: (v if not isinstance(v, torch.Tensor) else v.contiguous()) for k, v in kwargs.items()} 

235 

236 tensor = None 

237 for arg in args: 

238 if isinstance(arg, torch.Tensor): 

239 tensor = arg 

240 break 

241 if tensor is None: 

242 for value in kwargs.values(): 

243 if isinstance(value, torch.Tensor): 

244 tensor = value 

245 break 

246 

247 if tensor is not None: 

248 ctx = custom_device_ctx(tensor.device.index) 

249 else: 

250 ctx = contextlib.nullcontext() 

251 

252 with ctx: 

253 return fn(*contiguous_args, **contiguous_kwargs) 

254 

255 return wrapper 

256 

257 

258def _cpu_device_warning(): 

259 warnings.warn(('Triton is not supported on current platform, roll back to CPU.'), stacklevel=1) 

260 

261 

262@tensor_cache 

263def prepare_chunk_offsets(cu_seqlens: torch.LongTensor, chunk_size: int) -> torch.LongTensor: 

264 return torch.cat([cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]).cumsum(-1) 

265 

266 

267if os.environ.get('FLA_USE_FAST_OPS', '0') == '1': 

268 exp = tldevice.fast_expf 

269 exp2 = tldevice.exp2 

270 log = tldevice.fast_logf 

271 log2 = tldevice.fast_log2f 

272else: 

273 exp = tl.exp 

274 exp2 = tl.math.exp2 

275 log = tl.log 

276 log2 = tl.log2 

277 

278 

279def get_all_max_shared_mem(): 

280 try: 

281 return [ 

282 triton.runtime.driver.active.utils.get_device_properties(i)['max_shared_mem'] 

283 for i in range(device_torch_lib.device_count()) 

284 ] 

285 except Exception: 

286 _cpu_device_warning() 

287 return [-1] 

288 

289 

290class Backend(Enum): 

291 ADA = 101376 # RTX 4090 

292 AMPERE = 166912 # A100 

293 HOPPER = 232448 # H100 

294 DEFAULT = 102400 # Default 

295 

296 @classmethod 

297 def get_shared_memory(cls, arch: str) -> int: 

298 try: 

299 return cls[arch.upper()].value 

300 except KeyError: 

301 return cls.DEFAULT.value 

302 

303 

304@lru_cache(maxsize=None) 

305def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool: 

306 try: 

307 device_shared_mem_list = get_all_max_shared_mem() 

308 max_shared_memory = device_shared_mem_list[tensor_idx] 

309 return max_shared_memory >= Backend.get_shared_memory(arch) 

310 except Exception: 

311 return False 

312 

313 

314def get_autotune_config( 

315 multibuffer_list: tuple = (False,), 

316 unit_flag_list: tuple = (False,), 

317 limit_auto_multi_buffer_only_for_local_buffer_list: tuple = (False,), 

318 limit_auto_multi_buffer_of_local_buffer_list: tuple = ("no-l0c",), 

319 set_workspace_multibuffer_list: tuple = (2, 4), 

320 enable_hivm_auto_cv_balance_list: tuple = (True,), 

321 tile_mix_vector_loop_num_list: tuple = (2, 4), 

322 tile_mix_cube_loop_num_list: tuple = (2, 4), 

323): 

324 configs = [] 

325 for ( 

326 multibuffer, 

327 unit_flag, 

328 limit_auto_multi_buffer_only_for_local_buffer, 

329 limit_auto_multi_buffer_of_local_buffer, 

330 ) in itertools.product( 

331 list(multibuffer_list), 

332 list(unit_flag_list), 

333 list(limit_auto_multi_buffer_only_for_local_buffer_list), 

334 list(limit_auto_multi_buffer_of_local_buffer_list), 

335 ): 

336 base_config_dict = { 

337 'multibuffer': multibuffer, 

338 'unit_flag': unit_flag, 

339 'limit_auto_multi_buffer_only_for_local_buffer': limit_auto_multi_buffer_only_for_local_buffer, 

340 'limit_auto_multi_buffer_of_local_buffer': limit_auto_multi_buffer_of_local_buffer, 

341 } 

342 

343 if limit_auto_multi_buffer_only_for_local_buffer: 

344 configs.append(triton.Config(base_config_dict)) 

345 else: 

346 for ( 

347 set_workspace_multibuffer, 

348 enable_hivm_auto_cv_balance, 

349 tile_mix_vector_loop, 

350 tile_mix_cube_loop, 

351 ) in itertools.product( 

352 list(set_workspace_multibuffer_list), 

353 list(enable_hivm_auto_cv_balance_list), 

354 list(tile_mix_vector_loop_num_list), 

355 list(tile_mix_cube_loop_num_list), 

356 ): 

357 full_config_dict = base_config_dict.copy() 

358 full_config_dict.update( 

359 { 

360 'set_workspace_multibuffer': set_workspace_multibuffer, 

361 'enable_hivm_auto_cv_balance': enable_hivm_auto_cv_balance, 

362 'tile_mix_vector_loop': tile_mix_vector_loop, 

363 'tile_mix_cube_loop': tile_mix_cube_loop, 

364 } 

365 ) 

366 configs.append(triton.Config(full_config_dict)) 

367 return configs 

368 

369 

370def get_npu_properties(): 

371 return driver.active.utils.get_device_properties(torch.npu.current_device()) 

372 

373 

374@functools.cache 

375def get_vector_num() -> int: 

376 import torch_npu 

377 

378 current_device = torch_npu.npu.current_device() 

379 properties = driver.active.utils.get_device_properties(current_device) 

380 return properties["num_vectorcore"] 

381 

382 

383@lru_cache 

384def is_arch35(): 

385 try: 

386 import torch_npu 

387 

388 return "Ascend910_95" in torch_npu.npu.get_device_name() or "Ascend950" in torch_npu.npu.get_device_name() 

389 except Exception: 

390 return False