Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / gated_delta_net.py: 0%

153 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16 

17# Keep the validated kernel adapter close to its upstream implementation. 

18# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring 

19# pylint: disable=non-google-docstring,disallowed-name,unused-argument,invalid-name 

20# pylint: disable=missing-module-docstring,missing-function-docstring 

21# pylint: disable=abstract-method,arguments-differ 

22 

23import warnings 

24from typing import Optional 

25 

26import torch 

27 

28from ._triton.gated_delta_net.chunk_delta_h import chunk_gated_delta_rule_bwd_dhu, chunk_gated_delta_rule_fwd_h 

29from ._triton.gated_delta_net.chunk_o import chunk_bwd_dqkwg, chunk_bwd_dv_local, chunk_fwd_o 

30from ._triton.gated_delta_net.chunk_scaled_dot_kkt import chunk_scaled_dot_kkt_fwd 

31from ._triton.gated_delta_net.cumsum import chunk_local_cumsum 

32from ._triton.gated_delta_net.solve_tril import solve_tril 

33from ._triton.gated_delta_net.utils import autocast_custom_bwd, autocast_custom_fwd, input_guard 

34from ._triton.gated_delta_net.wy_fast import prepare_wy_repr_bwd, recompute_w_u_fwd 

35 

36 

37def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> tuple[torch.Tensor, torch.Tensor]: 

38 inv_norm = torch.rsqrt((x * x).sum(dim=-1, keepdim=True) + eps) 

39 return (x * inv_norm).to(x.dtype), inv_norm 

40 

41 

42def chunk_gated_delta_rule_fwd_prepare( 

43 k: torch.Tensor, 

44 v: torch.Tensor, 

45 g: torch.Tensor, 

46 beta: torch.Tensor, 

47 cu_seqlens: Optional[torch.LongTensor] = None, 

48 chunk_size: int = 64, 

49): 

50 """Compute forward intermediates that do not depend on the initial state.""" 

51 g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens, head_first=False) 

52 matrix_a = chunk_scaled_dot_kkt_fwd( 

53 k=k, 

54 g=g, 

55 beta=beta, 

56 cu_seqlens=cu_seqlens, 

57 chunk_size=chunk_size, 

58 output_dtype=torch.float32, 

59 ) 

60 matrix_a = solve_tril(A=matrix_a, cu_seqlens=cu_seqlens, output_dtype=k.dtype) 

61 w, u = recompute_w_u_fwd( 

62 k=k, 

63 v=v, 

64 beta=beta, 

65 A=matrix_a, 

66 g=g, 

67 cu_seqlens=cu_seqlens, 

68 ) 

69 return g, matrix_a, w, u 

70 

71 

72def chunk_gated_delta_rule_fwd_apply_state( 

73 k: torch.Tensor, 

74 g: torch.Tensor, 

75 w: torch.Tensor, 

76 u: torch.Tensor, 

77 initial_state: Optional[torch.Tensor], 

78 output_final_state: bool, 

79 cu_seqlens: Optional[torch.LongTensor] = None, 

80 chunk_size: int = 64, 

81): 

82 """Apply the recurrent initial state and return local state intermediates.""" 

83 return chunk_gated_delta_rule_fwd_h( 

84 k=k, 

85 w=w, 

86 u=u, 

87 g=g, 

88 initial_state=initial_state, 

89 output_final_state=output_final_state, 

90 chunk_size=chunk_size, 

91 cu_seqlens=cu_seqlens, 

92 ) 

93 

94 

95def chunk_gated_delta_rule_fwd_output( 

96 q: torch.Tensor, 

97 k: torch.Tensor, 

98 v_new: torch.Tensor, 

99 h: torch.Tensor, 

100 g: torch.Tensor, 

101 scale: float, 

102 cu_seqlens: Optional[torch.LongTensor] = None, 

103 chunk_size: int = 64, 

104): 

105 """Compute local outputs after recurrent states have been applied.""" 

106 return chunk_fwd_o( 

107 q=q, 

108 k=k, 

109 v=v_new, 

110 h=h, 

111 g=g, 

112 scale=scale, 

113 cu_seqlens=cu_seqlens, 

114 chunk_size=chunk_size, 

115 ) 

116 

117 

118def chunk_gated_delta_rule_fwd( 

119 q: torch.Tensor, 

120 k: torch.Tensor, 

121 v: torch.Tensor, 

122 g: torch.Tensor, 

123 beta: torch.Tensor, 

124 scale: float, 

125 initial_state: torch.Tensor, 

126 output_final_state: bool, 

127 cu_seqlens: Optional[torch.LongTensor] = None, 

128 chunk_size: int = 64, 

129): 

130 g, matrix_a, w, u = chunk_gated_delta_rule_fwd_prepare( 

131 k=k, 

132 v=v, 

133 g=g, 

134 beta=beta, 

135 cu_seqlens=cu_seqlens, 

136 chunk_size=chunk_size, 

137 ) 

138 h, v_new, final_state = chunk_gated_delta_rule_fwd_apply_state( 

139 k=k, 

140 g=g, 

141 w=w, 

142 u=u, 

143 initial_state=initial_state, 

144 output_final_state=output_final_state, 

145 cu_seqlens=cu_seqlens, 

146 chunk_size=chunk_size, 

147 ) 

148 output = chunk_gated_delta_rule_fwd_output( 

149 q=q, 

150 k=k, 

151 v_new=v_new, 

152 h=h, 

153 g=g, 

154 scale=scale, 

155 cu_seqlens=cu_seqlens, 

156 chunk_size=chunk_size, 

157 ) 

158 return g, output, matrix_a, final_state 

159 

160 

161def chunk_gated_delta_rule_bwd_prepare( 

162 q: torch.Tensor, 

163 k: torch.Tensor, 

164 v: torch.Tensor, 

165 g: torch.Tensor, 

166 beta: torch.Tensor, 

167 A: torch.Tensor, 

168 scale: float, 

169 initial_state: Optional[torch.Tensor], 

170 do: torch.Tensor, 

171 cu_seqlens: Optional[torch.LongTensor] = None, 

172 chunk_size: int = 64, 

173): 

174 """Compute backward intermediates that do not depend on final-state grad.""" 

175 w, u = recompute_w_u_fwd( 

176 k=k, 

177 v=v, 

178 beta=beta, 

179 A=A, 

180 g=g, 

181 cu_seqlens=cu_seqlens, 

182 ) 

183 h, v_new, _ = chunk_gated_delta_rule_fwd_apply_state( 

184 k=k, 

185 g=g, 

186 w=w, 

187 u=u, 

188 initial_state=initial_state, 

189 output_final_state=False, 

190 cu_seqlens=cu_seqlens, 

191 chunk_size=chunk_size, 

192 ) 

193 dv = chunk_bwd_dv_local( 

194 q=q, 

195 k=k, 

196 g=g, 

197 do=do, 

198 scale=scale, 

199 cu_seqlens=cu_seqlens, 

200 chunk_size=chunk_size, 

201 ) 

202 return w, h, v_new, dv 

203 

204 

205def chunk_gated_delta_rule_bwd_state( 

206 q: torch.Tensor, 

207 k: torch.Tensor, 

208 w: torch.Tensor, 

209 g: torch.Tensor, 

210 initial_state: Optional[torch.Tensor], 

211 dht: Optional[torch.Tensor], 

212 do: torch.Tensor, 

213 dv: torch.Tensor, 

214 scale: float, 

215 cu_seqlens: Optional[torch.LongTensor] = None, 

216 chunk_size: int = 64, 

217): 

218 """Apply the final-state gradient and produce the initial-state gradient.""" 

219 return chunk_gated_delta_rule_bwd_dhu( 

220 q=q, 

221 k=k, 

222 w=w, 

223 g=g, 

224 h0=initial_state, 

225 dht=dht, 

226 do=do, 

227 dv=dv, 

228 scale=scale, 

229 cu_seqlens=cu_seqlens, 

230 chunk_size=chunk_size, 

231 ) 

232 

233 

234def chunk_gated_delta_rule_bwd_finish( 

235 q: torch.Tensor, 

236 k: torch.Tensor, 

237 v: torch.Tensor, 

238 g: torch.Tensor, 

239 beta: torch.Tensor, 

240 A: torch.Tensor, 

241 w: torch.Tensor, 

242 h: torch.Tensor, 

243 v_new: torch.Tensor, 

244 dv: torch.Tensor, 

245 do: torch.Tensor, 

246 dh: torch.Tensor, 

247 scale: float, 

248 cu_seqlens: Optional[torch.LongTensor] = None, 

249 chunk_size: int = 64, 

250): 

251 """Finish local tensor gradients after the state-gradient handoff.""" 

252 dq, dk, dw, dg = chunk_bwd_dqkwg( 

253 q=q, 

254 k=k, 

255 v=v_new, 

256 w=w, 

257 g=g, 

258 h=h, 

259 dv=dv, 

260 do=do, 

261 dh=dh, 

262 chunk_size=chunk_size, 

263 scale=scale, 

264 cu_seqlens=cu_seqlens, 

265 ) 

266 dk2, dv, db, dg2 = prepare_wy_repr_bwd( 

267 k=k, 

268 v=v, 

269 beta=beta, 

270 g=g, 

271 A=A, 

272 dw=dw, 

273 du=dv, 

274 cu_seqlens=cu_seqlens, 

275 chunk_size=chunk_size, 

276 ) 

277 dk.add_(dk2) 

278 dg.add_(dg2) 

279 if dg.dtype != torch.float32: 

280 raise ValueError(f"dg current type is {dg.dtype} , should be float32") 

281 dg = chunk_local_cumsum( 

282 dg, 

283 chunk_size=chunk_size, 

284 reverse=True, 

285 cu_seqlens=cu_seqlens, 

286 head_first=False, 

287 ) 

288 return dq, dk, dv, db, dg 

289 

290 

291def chunk_gated_delta_rule_bwd( 

292 q: torch.Tensor, 

293 k: torch.Tensor, 

294 v: torch.Tensor, 

295 g: torch.Tensor, 

296 beta: torch.Tensor, 

297 A: torch.Tensor, 

298 scale: float, 

299 initial_state: torch.Tensor, 

300 do: torch.Tensor, 

301 dht: torch.Tensor, 

302 cu_seqlens: Optional[torch.LongTensor] = None, 

303 chunk_size: int = 64, 

304): 

305 w, h, v_new, dv = chunk_gated_delta_rule_bwd_prepare( 

306 q=q, 

307 k=k, 

308 v=v, 

309 g=g, 

310 beta=beta, 

311 A=A, 

312 scale=scale, 

313 initial_state=initial_state, 

314 do=do, 

315 cu_seqlens=cu_seqlens, 

316 chunk_size=chunk_size, 

317 ) 

318 dh, dh0, dv = chunk_gated_delta_rule_bwd_state( 

319 q=q, 

320 k=k, 

321 w=w, 

322 g=g, 

323 initial_state=initial_state, 

324 dht=dht, 

325 do=do, 

326 dv=dv, 

327 scale=scale, 

328 cu_seqlens=cu_seqlens, 

329 chunk_size=chunk_size, 

330 ) 

331 dq, dk, dv, db, dg = chunk_gated_delta_rule_bwd_finish( 

332 q=q, 

333 k=k, 

334 v=v, 

335 g=g, 

336 beta=beta, 

337 A=A, 

338 w=w, 

339 h=h, 

340 v_new=v_new, 

341 dv=dv, 

342 do=do, 

343 dh=dh, 

344 scale=scale, 

345 cu_seqlens=cu_seqlens, 

346 chunk_size=chunk_size, 

347 ) 

348 return dq, dk, dv, db, dg, dh0 

349 

350 

351@torch.compiler.disable 

352@input_guard 

353def chunk_gated_delta_rule_fwd_prepare_saved( 

354 q: torch.Tensor, 

355 k: torch.Tensor, 

356 v: torch.Tensor, 

357 g: torch.Tensor, 

358 beta: torch.Tensor, 

359 scale: float = None, 

360 use_qk_l2norm_in_kernel: bool = False, 

361 chunk_size: int = 64, 

362 cu_seqlens: torch.Tensor = None, 

363): 

364 """Prepare fused GDN forward tensors without consuming the initial state.""" 

365 if scale is None: 

366 scale = k.shape[-1] ** -0.5 

367 

368 q_norm, q_inv_norm = q, q.new_empty(0) 

369 k_norm, k_inv_norm = k, k.new_empty(0) 

370 if use_qk_l2norm_in_kernel: 

371 q_norm, q_inv_norm = _l2norm(q) 

372 k_norm, k_inv_norm = _l2norm(k) 

373 

374 g_cumsum, matrix_a, w, u = chunk_gated_delta_rule_fwd_prepare( 

375 k=k_norm, 

376 v=v, 

377 g=g, 

378 beta=beta, 

379 chunk_size=chunk_size, 

380 cu_seqlens=cu_seqlens, 

381 ) 

382 return ( 

383 q_norm, 

384 k_norm, 

385 q_inv_norm, 

386 k_inv_norm, 

387 g_cumsum, 

388 matrix_a, 

389 w, 

390 u, 

391 scale, 

392 ) 

393 

394 

395@torch.compiler.disable 

396@input_guard 

397def chunk_gated_delta_rule_fwd_apply_state_saved( 

398 k_norm: torch.Tensor, 

399 g_cumsum: torch.Tensor, 

400 w: torch.Tensor, 

401 u: torch.Tensor, 

402 initial_state: torch.Tensor = None, 

403 output_final_state: bool = True, 

404 chunk_size: int = 64, 

405 cu_seqlens: torch.Tensor = None, 

406): 

407 """Apply an initial state to fused prepared forward tensors.""" 

408 return chunk_gated_delta_rule_fwd_apply_state( 

409 k=k_norm, 

410 g=g_cumsum, 

411 w=w, 

412 u=u, 

413 initial_state=initial_state, 

414 output_final_state=output_final_state, 

415 chunk_size=chunk_size, 

416 cu_seqlens=cu_seqlens, 

417 ) 

418 

419 

420@torch.compiler.disable 

421@input_guard 

422def chunk_gated_delta_rule_fwd_output_saved( 

423 q_norm: torch.Tensor, 

424 k_norm: torch.Tensor, 

425 g_cumsum: torch.Tensor, 

426 h: torch.Tensor, 

427 v_new: torch.Tensor, 

428 scale: float, 

429 chunk_size: int = 64, 

430 cu_seqlens: torch.Tensor = None, 

431): 

432 """Compute fused GDN output from prepared, state-applied tensors.""" 

433 return chunk_gated_delta_rule_fwd_output( 

434 q=q_norm, 

435 k=k_norm, 

436 v_new=v_new, 

437 h=h, 

438 g=g_cumsum, 

439 scale=scale, 

440 chunk_size=chunk_size, 

441 cu_seqlens=cu_seqlens, 

442 ) 

443 

444 

445@torch.compiler.disable 

446@input_guard 

447def chunk_gated_delta_rule_fwd_saved( 

448 q: torch.Tensor, 

449 k: torch.Tensor, 

450 v: torch.Tensor, 

451 g: torch.Tensor, 

452 beta: torch.Tensor, 

453 scale: float = None, 

454 initial_state: torch.Tensor = None, 

455 use_qk_l2norm_in_kernel: bool = False, 

456 chunk_size: int = 64, 

457 cu_seqlens: torch.Tensor = None, 

458): 

459 """Run fused GDN forward and return the tensors required by its backward.""" 

460 ( 

461 q_norm, 

462 k_norm, 

463 q_inv_norm, 

464 k_inv_norm, 

465 g_cumsum, 

466 matrix_a, 

467 w, 

468 u, 

469 scale, 

470 ) = chunk_gated_delta_rule_fwd_prepare_saved( 

471 q, 

472 k, 

473 v, 

474 g, 

475 beta, 

476 scale=scale, 

477 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, 

478 chunk_size=chunk_size, 

479 cu_seqlens=cu_seqlens, 

480 ) 

481 h, v_new, final_state = chunk_gated_delta_rule_fwd_apply_state_saved( 

482 k_norm, 

483 g_cumsum, 

484 w, 

485 u, 

486 initial_state=initial_state, 

487 output_final_state=True, 

488 chunk_size=chunk_size, 

489 cu_seqlens=cu_seqlens, 

490 ) 

491 output = chunk_gated_delta_rule_fwd_output_saved( 

492 q_norm, 

493 k_norm, 

494 g_cumsum, 

495 h, 

496 v_new, 

497 scale, 

498 chunk_size=chunk_size, 

499 cu_seqlens=cu_seqlens, 

500 ) 

501 return ( 

502 output.to(q.dtype), 

503 final_state, 

504 q_norm, 

505 k_norm, 

506 q_inv_norm, 

507 k_inv_norm, 

508 g_cumsum, 

509 matrix_a, 

510 scale, 

511 ) 

512 

513 

514@torch.compiler.disable 

515@input_guard 

516def chunk_gated_delta_rule_bwd_prepare_saved( 

517 q_norm: torch.Tensor, 

518 k_norm: torch.Tensor, 

519 v: torch.Tensor, 

520 g_cumsum: torch.Tensor, 

521 beta: torch.Tensor, 

522 A: torch.Tensor, 

523 initial_state: torch.Tensor, 

524 grad_output: torch.Tensor, 

525 scale: float, 

526 chunk_size: int = 64, 

527 cu_seqlens: torch.Tensor = None, 

528): 

529 """Prepare fused backward tensors before the final-state grad arrives.""" 

530 return chunk_gated_delta_rule_bwd_prepare( 

531 q=q_norm, 

532 k=k_norm, 

533 v=v, 

534 g=g_cumsum, 

535 beta=beta, 

536 A=A, 

537 scale=scale, 

538 initial_state=initial_state, 

539 do=grad_output, 

540 chunk_size=chunk_size, 

541 cu_seqlens=cu_seqlens, 

542 ) 

543 

544 

545@torch.compiler.disable 

546@input_guard 

547def chunk_gated_delta_rule_bwd_state_saved( 

548 q_norm: torch.Tensor, 

549 k_norm: torch.Tensor, 

550 g_cumsum: torch.Tensor, 

551 w: torch.Tensor, 

552 initial_state: torch.Tensor, 

553 grad_final_state: torch.Tensor, 

554 grad_output: torch.Tensor, 

555 dv_local: torch.Tensor, 

556 scale: float, 

557 chunk_size: int = 64, 

558 cu_seqlens: torch.Tensor = None, 

559): 

560 """Consume the final-state grad and produce the initial-state grad.""" 

561 return chunk_gated_delta_rule_bwd_state( 

562 q=q_norm, 

563 k=k_norm, 

564 w=w, 

565 g=g_cumsum, 

566 initial_state=initial_state, 

567 dht=grad_final_state, 

568 do=grad_output, 

569 dv=dv_local, 

570 scale=scale, 

571 chunk_size=chunk_size, 

572 cu_seqlens=cu_seqlens, 

573 ) 

574 

575 

576@torch.compiler.disable 

577@input_guard 

578def chunk_gated_delta_rule_bwd_finish_saved( 

579 q: torch.Tensor, 

580 k: torch.Tensor, 

581 q_norm: torch.Tensor, 

582 k_norm: torch.Tensor, 

583 v: torch.Tensor, 

584 g_cumsum: torch.Tensor, 

585 beta: torch.Tensor, 

586 A: torch.Tensor, 

587 w: torch.Tensor, 

588 h: torch.Tensor, 

589 v_new: torch.Tensor, 

590 dv: torch.Tensor, 

591 grad_output: torch.Tensor, 

592 dh: torch.Tensor, 

593 q_inv_norm: torch.Tensor, 

594 k_inv_norm: torch.Tensor, 

595 scale: float, 

596 use_qk_l2norm_in_kernel: bool = False, 

597 chunk_size: int = 64, 

598 cu_seqlens: torch.Tensor = None, 

599): 

600 """Finish local fused gradients after the P2P state-gradient handoff.""" 

601 dq, dk, dv, dbeta, dg = chunk_gated_delta_rule_bwd_finish( 

602 q=q_norm, 

603 k=k_norm, 

604 v=v, 

605 g=g_cumsum, 

606 beta=beta, 

607 A=A, 

608 w=w, 

609 h=h, 

610 v_new=v_new, 

611 dv=dv, 

612 do=grad_output, 

613 dh=dh, 

614 scale=scale, 

615 chunk_size=chunk_size, 

616 cu_seqlens=cu_seqlens, 

617 ) 

618 if use_qk_l2norm_in_kernel: 

619 with torch.enable_grad(): 

620 q_leaf = q.detach().requires_grad_(True) 

621 k_leaf = k.detach().requires_grad_(True) 

622 q_recomputed, _ = _l2norm(q_leaf) 

623 k_recomputed, _ = _l2norm(k_leaf) 

624 dq, dk = torch.autograd.grad( 

625 (q_recomputed, k_recomputed), 

626 (q_leaf, k_leaf), 

627 grad_outputs=(dq, dk), 

628 ) 

629 del q_inv_norm, k_inv_norm 

630 return dq, dk, dv, dg, dbeta 

631 

632 

633@torch.compiler.disable 

634@input_guard 

635def chunk_gated_delta_rule_bwd_saved( 

636 q: torch.Tensor, 

637 k: torch.Tensor, 

638 q_norm: torch.Tensor, 

639 k_norm: torch.Tensor, 

640 v: torch.Tensor, 

641 g_cumsum: torch.Tensor, 

642 beta: torch.Tensor, 

643 A: torch.Tensor, 

644 initial_state: torch.Tensor, 

645 grad_output: torch.Tensor, 

646 grad_final_state: torch.Tensor, 

647 q_inv_norm: torch.Tensor, 

648 k_inv_norm: torch.Tensor, 

649 scale: float, 

650 use_qk_l2norm_in_kernel: bool = False, 

651 chunk_size: int = 64, 

652 cu_seqlens: torch.Tensor = None, 

653): 

654 """Run fused GDN backward from a context saved by the forward helper.""" 

655 w, h, v_new, dv = chunk_gated_delta_rule_bwd_prepare_saved( 

656 q_norm, 

657 k_norm, 

658 v, 

659 g_cumsum, 

660 beta, 

661 A, 

662 initial_state, 

663 grad_output, 

664 scale, 

665 chunk_size=chunk_size, 

666 cu_seqlens=cu_seqlens, 

667 ) 

668 dh, dh0, dv = chunk_gated_delta_rule_bwd_state_saved( 

669 q_norm, 

670 k_norm, 

671 g_cumsum, 

672 w, 

673 initial_state, 

674 grad_final_state, 

675 grad_output, 

676 dv, 

677 scale, 

678 chunk_size=chunk_size, 

679 cu_seqlens=cu_seqlens, 

680 ) 

681 dq, dk, dv, dg, dbeta = chunk_gated_delta_rule_bwd_finish_saved( 

682 q, 

683 k, 

684 q_norm, 

685 k_norm, 

686 v, 

687 g_cumsum, 

688 beta, 

689 A, 

690 w, 

691 h, 

692 v_new, 

693 dv, 

694 grad_output, 

695 dh, 

696 q_inv_norm, 

697 k_inv_norm, 

698 scale, 

699 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, 

700 chunk_size=chunk_size, 

701 cu_seqlens=cu_seqlens, 

702 ) 

703 return dq, dk, dv, dg, dbeta, dh0 

704 

705 

706class ChunkGatedDeltaRuleFunction(torch.autograd.Function): 

707 """Autograd wrapper for the Triton-Ascend chunk Gated Delta Rule.""" 

708 

709 @staticmethod 

710 @input_guard 

711 @autocast_custom_fwd 

712 def forward( 

713 ctx, 

714 q: torch.Tensor, 

715 k: torch.Tensor, 

716 v: torch.Tensor, 

717 g: torch.Tensor, 

718 beta: torch.Tensor, 

719 scale: float, 

720 initial_state: torch.Tensor, 

721 output_final_state: bool, 

722 cu_seqlens: Optional[torch.LongTensor] = None, 

723 use_qk_l2norm_in_kernel: bool = False, 

724 chunk_size: int = 64, 

725 ): 

726 g, output, matrix_a, final_state = chunk_gated_delta_rule_fwd( 

727 q=q, 

728 k=k, 

729 v=v, 

730 g=g, 

731 beta=beta, 

732 scale=scale, 

733 initial_state=initial_state, 

734 output_final_state=output_final_state, 

735 cu_seqlens=cu_seqlens, 

736 chunk_size=chunk_size, 

737 ) 

738 

739 saved_initial_state = initial_state if initial_state is not None else q.new_empty(0) 

740 saved_cu_seqlens = cu_seqlens if cu_seqlens is not None else q.new_empty(0, dtype=torch.long) 

741 ctx.save_for_backward(q, k, v, g, beta, matrix_a, saved_initial_state, saved_cu_seqlens) 

742 ctx.has_initial_state = initial_state is not None 

743 ctx.has_cu_seqlens = cu_seqlens is not None 

744 ctx.scale = scale 

745 ctx.chunk_size = chunk_size 

746 return output.to(q.dtype), final_state 

747 

748 @staticmethod 

749 @input_guard 

750 @autocast_custom_bwd 

751 def backward( 

752 ctx, 

753 do: torch.Tensor, 

754 dht: torch.Tensor 

755 ): 

756 q, k, v, g, beta, matrix_a, initial_state, cu_seqlens = ctx.saved_tensors 

757 if not ctx.has_initial_state: 

758 initial_state = None 

759 if not ctx.has_cu_seqlens: 

760 cu_seqlens = None 

761 dq, dk, dv, db, dg, dh0 = chunk_gated_delta_rule_bwd( 

762 q=q, 

763 k=k, 

764 v=v, 

765 g=g, 

766 beta=beta, 

767 A=matrix_a, 

768 scale=ctx.scale, 

769 initial_state=initial_state, 

770 do=do, 

771 dht=dht, 

772 cu_seqlens=cu_seqlens, 

773 chunk_size=ctx.chunk_size, 

774 ) 

775 return dq.to(q), dk.to(k), dv.to(v), dg.to(g), db.to(beta), None, dh0, None, None, None, None 

776 

777 

778@torch.compiler.disable 

779def chunk_gated_delta_rule( 

780 q: torch.Tensor, 

781 k: torch.Tensor, 

782 v: torch.Tensor, 

783 g: torch.Tensor, 

784 beta: torch.Tensor, 

785 scale: float = None, 

786 initial_state: torch.Tensor = None, 

787 output_final_state: bool = False, 

788 use_qk_l2norm_in_kernel: bool = False, 

789 cu_seqlens: Optional[torch.LongTensor] = None, 

790 chunk_size: int = 64, 

791 head_first: bool = False, 

792): 

793 r""" 

794 Args: 

795 q (torch.Tensor): 

796 queries of shape `[B, T, H, K]`. 

797 k (torch.Tensor): 

798 keys of shape `[B, T, H, K]`. 

799 v (torch.Tensor): 

800 values of shape `[B, T, H, V]`. 

801 g (torch.Tensor): 

802 (forget) gating tensor (in log space!) of shape `[B, T, H]`. 

803 beta (torch.Tensor): 

804 betas of shape `[B, T, H]`. 

805 scale (Optional[float]): 

806 Scale factor for the RetNet attention scores. 

807 If not provided, it will default to `1 / sqrt(K)`. Default: `None`. 

808 initial_state (Optional[torch.Tensor]): 

809 Initial state of shape `[N, H, K, V]` for `N` input sequences. 

810 For equal-length input sequences, `N` equals the batch size `B`. 

811 Default: `None`. 

812 output_final_state (Optional[bool]): 

813 Whether to output the final state of shape `[N, H, K, V]`. Default: `False`. 

814 use_qk_l2norm_in_kernel (bool): 

815 Whether to apply L2norm to the q/k tensor internally. Default: `False`. 

816 cu_seqlens (torch.LongTensor): 

817 Cumulative sequence lengths of shape `[N+1]` used for variable-length training, 

818 consistent with the FlashAttention API. 

819 head_first (Optional[bool]): 

820 Whether the inputs are in the head-first format. Default: `False`. 

821 This argument has been deprecated. 

822 Returns: 

823 o (torch.Tensor): 

824 Outputs of shape `[B, T, H, V]`. 

825 final_state (torch.Tensor): 

826 Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`. 

827 

828 Examples:: 

829 >>> import torch 

830 >>> import torch.nn.functional as F 

831 >>> from einops import rearrange 

832 >>> from fla.ops.gated_delta_rule import chunk_gated_delta_rule 

833 # inputs with equal lengths 

834 >>> B, T, H, K, V = 4, 2048, 4, 512, 512 

835 >>> q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda') 

836 >>> k = F.normalize(torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda'), p=2, dim=-1) 

837 >>> v = torch.randn(B, T, H, V, dtype=torch.bfloat16, device='cuda') 

838 >>> beta = torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda').sigmoid() 

839 >>> g = F.logsigmoid(torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda')) 

840 >>> h0 = torch.randn(B, H, K, V, dtype=torch.bfloat16, device='cuda') 

841 >>> o, ht = chunk_gated_delta_rule( 

842 q, k, v, g, beta, 

843 initial_state=h0, 

844 output_final_state=True 

845 ) 

846 # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required 

847 >>> q, k, v, beta, g = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, beta, g)) 

848 # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected 

849 >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.long) 

850 >>> o, ht = chunk_gated_delta_rule( 

851 q, k, v, g, beta, 

852 initial_state=h0, 

853 output_final_state=True, 

854 cu_seqlens=cu_seqlens 

855 ) 

856 """ 

857 if q.dtype != k.dtype or k.dtype != v.dtype: 

858 raise ValueError( 

859 f"q current type is {q.dtype}, k current type is {k.dtype}, " 

860 f"v current type is {v.dtype}, they should be equal" 

861 ) 

862 if q.dtype == torch.float32: 

863 raise ValueError( 

864 "ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16." 

865 ) 

866 if len(beta.shape) != 3: 

867 raise ValueError( 

868 f"beta current shape len is {len(beta.shape)}, beta must be of shape [B, T, H] if head_first=False, or [B, H, T] otherwise." 

869 ) 

870 

871 if head_first: 

872 warnings.warn( 

873 "head_first is deprecated and will be removed in a future version. " 

874 "Please use head_first=False for now instead." 

875 ) 

876 if not head_first and q.shape[1] < q.shape[2]: 

877 warnings.warn( 

878 f"Input tensor shape suggests potential format mismatch: seq_len ({q.shape[1]}) < num_heads ({q.shape[2]}). " 

879 "This may indicate the inputs were passed in head-first format [B, H, T, ...] " 

880 "when head_first=False was specified. " 

881 "Please verify your input tensor format matches the expected shape [B, T, H, ...]." 

882 ) 

883 if cu_seqlens is not None: 

884 if q.shape[0] != 1: 

885 raise ValueError( 

886 f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." 

887 f"Please flatten variable-length inputs before processing." 

888 ) 

889 if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1: 

890 raise ValueError( 

891 f"The number of initial states is expected to be equal to the number of input sequences, " 

892 f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}." 

893 ) 

894 if scale is None: 

895 scale = k.shape[-1] ** -0.5 

896 

897 if use_qk_l2norm_in_kernel: 

898 q, _ = _l2norm(q) 

899 k, _ = _l2norm(k) 

900 

901 output, final_state = ChunkGatedDeltaRuleFunction.apply( 

902 q, 

903 k, 

904 v, 

905 g, 

906 beta, 

907 scale, 

908 initial_state, 

909 output_final_state, 

910 cu_seqlens, 

911 False, 

912 chunk_size, 

913 ) 

914 return output, final_state