Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / gated_delta_net.py: 0%
153 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
17# Keep the validated kernel adapter close to its upstream implementation.
18# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring
19# pylint: disable=non-google-docstring,disallowed-name,unused-argument,invalid-name
20# pylint: disable=missing-module-docstring,missing-function-docstring
21# pylint: disable=abstract-method,arguments-differ
23import warnings
24from typing import Optional
26import torch
28from ._triton.gated_delta_net.chunk_delta_h import chunk_gated_delta_rule_bwd_dhu, chunk_gated_delta_rule_fwd_h
29from ._triton.gated_delta_net.chunk_o import chunk_bwd_dqkwg, chunk_bwd_dv_local, chunk_fwd_o
30from ._triton.gated_delta_net.chunk_scaled_dot_kkt import chunk_scaled_dot_kkt_fwd
31from ._triton.gated_delta_net.cumsum import chunk_local_cumsum
32from ._triton.gated_delta_net.solve_tril import solve_tril
33from ._triton.gated_delta_net.utils import autocast_custom_bwd, autocast_custom_fwd, input_guard
34from ._triton.gated_delta_net.wy_fast import prepare_wy_repr_bwd, recompute_w_u_fwd
37def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> tuple[torch.Tensor, torch.Tensor]:
38 inv_norm = torch.rsqrt((x * x).sum(dim=-1, keepdim=True) + eps)
39 return (x * inv_norm).to(x.dtype), inv_norm
42def chunk_gated_delta_rule_fwd_prepare(
43 k: torch.Tensor,
44 v: torch.Tensor,
45 g: torch.Tensor,
46 beta: torch.Tensor,
47 cu_seqlens: Optional[torch.LongTensor] = None,
48 chunk_size: int = 64,
49):
50 """Compute forward intermediates that do not depend on the initial state."""
51 g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens, head_first=False)
52 matrix_a = chunk_scaled_dot_kkt_fwd(
53 k=k,
54 g=g,
55 beta=beta,
56 cu_seqlens=cu_seqlens,
57 chunk_size=chunk_size,
58 output_dtype=torch.float32,
59 )
60 matrix_a = solve_tril(A=matrix_a, cu_seqlens=cu_seqlens, output_dtype=k.dtype)
61 w, u = recompute_w_u_fwd(
62 k=k,
63 v=v,
64 beta=beta,
65 A=matrix_a,
66 g=g,
67 cu_seqlens=cu_seqlens,
68 )
69 return g, matrix_a, w, u
72def chunk_gated_delta_rule_fwd_apply_state(
73 k: torch.Tensor,
74 g: torch.Tensor,
75 w: torch.Tensor,
76 u: torch.Tensor,
77 initial_state: Optional[torch.Tensor],
78 output_final_state: bool,
79 cu_seqlens: Optional[torch.LongTensor] = None,
80 chunk_size: int = 64,
81):
82 """Apply the recurrent initial state and return local state intermediates."""
83 return chunk_gated_delta_rule_fwd_h(
84 k=k,
85 w=w,
86 u=u,
87 g=g,
88 initial_state=initial_state,
89 output_final_state=output_final_state,
90 chunk_size=chunk_size,
91 cu_seqlens=cu_seqlens,
92 )
95def chunk_gated_delta_rule_fwd_output(
96 q: torch.Tensor,
97 k: torch.Tensor,
98 v_new: torch.Tensor,
99 h: torch.Tensor,
100 g: torch.Tensor,
101 scale: float,
102 cu_seqlens: Optional[torch.LongTensor] = None,
103 chunk_size: int = 64,
104):
105 """Compute local outputs after recurrent states have been applied."""
106 return chunk_fwd_o(
107 q=q,
108 k=k,
109 v=v_new,
110 h=h,
111 g=g,
112 scale=scale,
113 cu_seqlens=cu_seqlens,
114 chunk_size=chunk_size,
115 )
118def chunk_gated_delta_rule_fwd(
119 q: torch.Tensor,
120 k: torch.Tensor,
121 v: torch.Tensor,
122 g: torch.Tensor,
123 beta: torch.Tensor,
124 scale: float,
125 initial_state: torch.Tensor,
126 output_final_state: bool,
127 cu_seqlens: Optional[torch.LongTensor] = None,
128 chunk_size: int = 64,
129):
130 g, matrix_a, w, u = chunk_gated_delta_rule_fwd_prepare(
131 k=k,
132 v=v,
133 g=g,
134 beta=beta,
135 cu_seqlens=cu_seqlens,
136 chunk_size=chunk_size,
137 )
138 h, v_new, final_state = chunk_gated_delta_rule_fwd_apply_state(
139 k=k,
140 g=g,
141 w=w,
142 u=u,
143 initial_state=initial_state,
144 output_final_state=output_final_state,
145 cu_seqlens=cu_seqlens,
146 chunk_size=chunk_size,
147 )
148 output = chunk_gated_delta_rule_fwd_output(
149 q=q,
150 k=k,
151 v_new=v_new,
152 h=h,
153 g=g,
154 scale=scale,
155 cu_seqlens=cu_seqlens,
156 chunk_size=chunk_size,
157 )
158 return g, output, matrix_a, final_state
161def chunk_gated_delta_rule_bwd_prepare(
162 q: torch.Tensor,
163 k: torch.Tensor,
164 v: torch.Tensor,
165 g: torch.Tensor,
166 beta: torch.Tensor,
167 A: torch.Tensor,
168 scale: float,
169 initial_state: Optional[torch.Tensor],
170 do: torch.Tensor,
171 cu_seqlens: Optional[torch.LongTensor] = None,
172 chunk_size: int = 64,
173):
174 """Compute backward intermediates that do not depend on final-state grad."""
175 w, u = recompute_w_u_fwd(
176 k=k,
177 v=v,
178 beta=beta,
179 A=A,
180 g=g,
181 cu_seqlens=cu_seqlens,
182 )
183 h, v_new, _ = chunk_gated_delta_rule_fwd_apply_state(
184 k=k,
185 g=g,
186 w=w,
187 u=u,
188 initial_state=initial_state,
189 output_final_state=False,
190 cu_seqlens=cu_seqlens,
191 chunk_size=chunk_size,
192 )
193 dv = chunk_bwd_dv_local(
194 q=q,
195 k=k,
196 g=g,
197 do=do,
198 scale=scale,
199 cu_seqlens=cu_seqlens,
200 chunk_size=chunk_size,
201 )
202 return w, h, v_new, dv
205def chunk_gated_delta_rule_bwd_state(
206 q: torch.Tensor,
207 k: torch.Tensor,
208 w: torch.Tensor,
209 g: torch.Tensor,
210 initial_state: Optional[torch.Tensor],
211 dht: Optional[torch.Tensor],
212 do: torch.Tensor,
213 dv: torch.Tensor,
214 scale: float,
215 cu_seqlens: Optional[torch.LongTensor] = None,
216 chunk_size: int = 64,
217):
218 """Apply the final-state gradient and produce the initial-state gradient."""
219 return chunk_gated_delta_rule_bwd_dhu(
220 q=q,
221 k=k,
222 w=w,
223 g=g,
224 h0=initial_state,
225 dht=dht,
226 do=do,
227 dv=dv,
228 scale=scale,
229 cu_seqlens=cu_seqlens,
230 chunk_size=chunk_size,
231 )
234def chunk_gated_delta_rule_bwd_finish(
235 q: torch.Tensor,
236 k: torch.Tensor,
237 v: torch.Tensor,
238 g: torch.Tensor,
239 beta: torch.Tensor,
240 A: torch.Tensor,
241 w: torch.Tensor,
242 h: torch.Tensor,
243 v_new: torch.Tensor,
244 dv: torch.Tensor,
245 do: torch.Tensor,
246 dh: torch.Tensor,
247 scale: float,
248 cu_seqlens: Optional[torch.LongTensor] = None,
249 chunk_size: int = 64,
250):
251 """Finish local tensor gradients after the state-gradient handoff."""
252 dq, dk, dw, dg = chunk_bwd_dqkwg(
253 q=q,
254 k=k,
255 v=v_new,
256 w=w,
257 g=g,
258 h=h,
259 dv=dv,
260 do=do,
261 dh=dh,
262 chunk_size=chunk_size,
263 scale=scale,
264 cu_seqlens=cu_seqlens,
265 )
266 dk2, dv, db, dg2 = prepare_wy_repr_bwd(
267 k=k,
268 v=v,
269 beta=beta,
270 g=g,
271 A=A,
272 dw=dw,
273 du=dv,
274 cu_seqlens=cu_seqlens,
275 chunk_size=chunk_size,
276 )
277 dk.add_(dk2)
278 dg.add_(dg2)
279 if dg.dtype != torch.float32:
280 raise ValueError(f"dg current type is {dg.dtype} , should be float32")
281 dg = chunk_local_cumsum(
282 dg,
283 chunk_size=chunk_size,
284 reverse=True,
285 cu_seqlens=cu_seqlens,
286 head_first=False,
287 )
288 return dq, dk, dv, db, dg
291def chunk_gated_delta_rule_bwd(
292 q: torch.Tensor,
293 k: torch.Tensor,
294 v: torch.Tensor,
295 g: torch.Tensor,
296 beta: torch.Tensor,
297 A: torch.Tensor,
298 scale: float,
299 initial_state: torch.Tensor,
300 do: torch.Tensor,
301 dht: torch.Tensor,
302 cu_seqlens: Optional[torch.LongTensor] = None,
303 chunk_size: int = 64,
304):
305 w, h, v_new, dv = chunk_gated_delta_rule_bwd_prepare(
306 q=q,
307 k=k,
308 v=v,
309 g=g,
310 beta=beta,
311 A=A,
312 scale=scale,
313 initial_state=initial_state,
314 do=do,
315 cu_seqlens=cu_seqlens,
316 chunk_size=chunk_size,
317 )
318 dh, dh0, dv = chunk_gated_delta_rule_bwd_state(
319 q=q,
320 k=k,
321 w=w,
322 g=g,
323 initial_state=initial_state,
324 dht=dht,
325 do=do,
326 dv=dv,
327 scale=scale,
328 cu_seqlens=cu_seqlens,
329 chunk_size=chunk_size,
330 )
331 dq, dk, dv, db, dg = chunk_gated_delta_rule_bwd_finish(
332 q=q,
333 k=k,
334 v=v,
335 g=g,
336 beta=beta,
337 A=A,
338 w=w,
339 h=h,
340 v_new=v_new,
341 dv=dv,
342 do=do,
343 dh=dh,
344 scale=scale,
345 cu_seqlens=cu_seqlens,
346 chunk_size=chunk_size,
347 )
348 return dq, dk, dv, db, dg, dh0
351@torch.compiler.disable
352@input_guard
353def chunk_gated_delta_rule_fwd_prepare_saved(
354 q: torch.Tensor,
355 k: torch.Tensor,
356 v: torch.Tensor,
357 g: torch.Tensor,
358 beta: torch.Tensor,
359 scale: float = None,
360 use_qk_l2norm_in_kernel: bool = False,
361 chunk_size: int = 64,
362 cu_seqlens: torch.Tensor = None,
363):
364 """Prepare fused GDN forward tensors without consuming the initial state."""
365 if scale is None:
366 scale = k.shape[-1] ** -0.5
368 q_norm, q_inv_norm = q, q.new_empty(0)
369 k_norm, k_inv_norm = k, k.new_empty(0)
370 if use_qk_l2norm_in_kernel:
371 q_norm, q_inv_norm = _l2norm(q)
372 k_norm, k_inv_norm = _l2norm(k)
374 g_cumsum, matrix_a, w, u = chunk_gated_delta_rule_fwd_prepare(
375 k=k_norm,
376 v=v,
377 g=g,
378 beta=beta,
379 chunk_size=chunk_size,
380 cu_seqlens=cu_seqlens,
381 )
382 return (
383 q_norm,
384 k_norm,
385 q_inv_norm,
386 k_inv_norm,
387 g_cumsum,
388 matrix_a,
389 w,
390 u,
391 scale,
392 )
395@torch.compiler.disable
396@input_guard
397def chunk_gated_delta_rule_fwd_apply_state_saved(
398 k_norm: torch.Tensor,
399 g_cumsum: torch.Tensor,
400 w: torch.Tensor,
401 u: torch.Tensor,
402 initial_state: torch.Tensor = None,
403 output_final_state: bool = True,
404 chunk_size: int = 64,
405 cu_seqlens: torch.Tensor = None,
406):
407 """Apply an initial state to fused prepared forward tensors."""
408 return chunk_gated_delta_rule_fwd_apply_state(
409 k=k_norm,
410 g=g_cumsum,
411 w=w,
412 u=u,
413 initial_state=initial_state,
414 output_final_state=output_final_state,
415 chunk_size=chunk_size,
416 cu_seqlens=cu_seqlens,
417 )
420@torch.compiler.disable
421@input_guard
422def chunk_gated_delta_rule_fwd_output_saved(
423 q_norm: torch.Tensor,
424 k_norm: torch.Tensor,
425 g_cumsum: torch.Tensor,
426 h: torch.Tensor,
427 v_new: torch.Tensor,
428 scale: float,
429 chunk_size: int = 64,
430 cu_seqlens: torch.Tensor = None,
431):
432 """Compute fused GDN output from prepared, state-applied tensors."""
433 return chunk_gated_delta_rule_fwd_output(
434 q=q_norm,
435 k=k_norm,
436 v_new=v_new,
437 h=h,
438 g=g_cumsum,
439 scale=scale,
440 chunk_size=chunk_size,
441 cu_seqlens=cu_seqlens,
442 )
445@torch.compiler.disable
446@input_guard
447def chunk_gated_delta_rule_fwd_saved(
448 q: torch.Tensor,
449 k: torch.Tensor,
450 v: torch.Tensor,
451 g: torch.Tensor,
452 beta: torch.Tensor,
453 scale: float = None,
454 initial_state: torch.Tensor = None,
455 use_qk_l2norm_in_kernel: bool = False,
456 chunk_size: int = 64,
457 cu_seqlens: torch.Tensor = None,
458):
459 """Run fused GDN forward and return the tensors required by its backward."""
460 (
461 q_norm,
462 k_norm,
463 q_inv_norm,
464 k_inv_norm,
465 g_cumsum,
466 matrix_a,
467 w,
468 u,
469 scale,
470 ) = chunk_gated_delta_rule_fwd_prepare_saved(
471 q,
472 k,
473 v,
474 g,
475 beta,
476 scale=scale,
477 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
478 chunk_size=chunk_size,
479 cu_seqlens=cu_seqlens,
480 )
481 h, v_new, final_state = chunk_gated_delta_rule_fwd_apply_state_saved(
482 k_norm,
483 g_cumsum,
484 w,
485 u,
486 initial_state=initial_state,
487 output_final_state=True,
488 chunk_size=chunk_size,
489 cu_seqlens=cu_seqlens,
490 )
491 output = chunk_gated_delta_rule_fwd_output_saved(
492 q_norm,
493 k_norm,
494 g_cumsum,
495 h,
496 v_new,
497 scale,
498 chunk_size=chunk_size,
499 cu_seqlens=cu_seqlens,
500 )
501 return (
502 output.to(q.dtype),
503 final_state,
504 q_norm,
505 k_norm,
506 q_inv_norm,
507 k_inv_norm,
508 g_cumsum,
509 matrix_a,
510 scale,
511 )
514@torch.compiler.disable
515@input_guard
516def chunk_gated_delta_rule_bwd_prepare_saved(
517 q_norm: torch.Tensor,
518 k_norm: torch.Tensor,
519 v: torch.Tensor,
520 g_cumsum: torch.Tensor,
521 beta: torch.Tensor,
522 A: torch.Tensor,
523 initial_state: torch.Tensor,
524 grad_output: torch.Tensor,
525 scale: float,
526 chunk_size: int = 64,
527 cu_seqlens: torch.Tensor = None,
528):
529 """Prepare fused backward tensors before the final-state grad arrives."""
530 return chunk_gated_delta_rule_bwd_prepare(
531 q=q_norm,
532 k=k_norm,
533 v=v,
534 g=g_cumsum,
535 beta=beta,
536 A=A,
537 scale=scale,
538 initial_state=initial_state,
539 do=grad_output,
540 chunk_size=chunk_size,
541 cu_seqlens=cu_seqlens,
542 )
545@torch.compiler.disable
546@input_guard
547def chunk_gated_delta_rule_bwd_state_saved(
548 q_norm: torch.Tensor,
549 k_norm: torch.Tensor,
550 g_cumsum: torch.Tensor,
551 w: torch.Tensor,
552 initial_state: torch.Tensor,
553 grad_final_state: torch.Tensor,
554 grad_output: torch.Tensor,
555 dv_local: torch.Tensor,
556 scale: float,
557 chunk_size: int = 64,
558 cu_seqlens: torch.Tensor = None,
559):
560 """Consume the final-state grad and produce the initial-state grad."""
561 return chunk_gated_delta_rule_bwd_state(
562 q=q_norm,
563 k=k_norm,
564 w=w,
565 g=g_cumsum,
566 initial_state=initial_state,
567 dht=grad_final_state,
568 do=grad_output,
569 dv=dv_local,
570 scale=scale,
571 chunk_size=chunk_size,
572 cu_seqlens=cu_seqlens,
573 )
576@torch.compiler.disable
577@input_guard
578def chunk_gated_delta_rule_bwd_finish_saved(
579 q: torch.Tensor,
580 k: torch.Tensor,
581 q_norm: torch.Tensor,
582 k_norm: torch.Tensor,
583 v: torch.Tensor,
584 g_cumsum: torch.Tensor,
585 beta: torch.Tensor,
586 A: torch.Tensor,
587 w: torch.Tensor,
588 h: torch.Tensor,
589 v_new: torch.Tensor,
590 dv: torch.Tensor,
591 grad_output: torch.Tensor,
592 dh: torch.Tensor,
593 q_inv_norm: torch.Tensor,
594 k_inv_norm: torch.Tensor,
595 scale: float,
596 use_qk_l2norm_in_kernel: bool = False,
597 chunk_size: int = 64,
598 cu_seqlens: torch.Tensor = None,
599):
600 """Finish local fused gradients after the P2P state-gradient handoff."""
601 dq, dk, dv, dbeta, dg = chunk_gated_delta_rule_bwd_finish(
602 q=q_norm,
603 k=k_norm,
604 v=v,
605 g=g_cumsum,
606 beta=beta,
607 A=A,
608 w=w,
609 h=h,
610 v_new=v_new,
611 dv=dv,
612 do=grad_output,
613 dh=dh,
614 scale=scale,
615 chunk_size=chunk_size,
616 cu_seqlens=cu_seqlens,
617 )
618 if use_qk_l2norm_in_kernel:
619 with torch.enable_grad():
620 q_leaf = q.detach().requires_grad_(True)
621 k_leaf = k.detach().requires_grad_(True)
622 q_recomputed, _ = _l2norm(q_leaf)
623 k_recomputed, _ = _l2norm(k_leaf)
624 dq, dk = torch.autograd.grad(
625 (q_recomputed, k_recomputed),
626 (q_leaf, k_leaf),
627 grad_outputs=(dq, dk),
628 )
629 del q_inv_norm, k_inv_norm
630 return dq, dk, dv, dg, dbeta
633@torch.compiler.disable
634@input_guard
635def chunk_gated_delta_rule_bwd_saved(
636 q: torch.Tensor,
637 k: torch.Tensor,
638 q_norm: torch.Tensor,
639 k_norm: torch.Tensor,
640 v: torch.Tensor,
641 g_cumsum: torch.Tensor,
642 beta: torch.Tensor,
643 A: torch.Tensor,
644 initial_state: torch.Tensor,
645 grad_output: torch.Tensor,
646 grad_final_state: torch.Tensor,
647 q_inv_norm: torch.Tensor,
648 k_inv_norm: torch.Tensor,
649 scale: float,
650 use_qk_l2norm_in_kernel: bool = False,
651 chunk_size: int = 64,
652 cu_seqlens: torch.Tensor = None,
653):
654 """Run fused GDN backward from a context saved by the forward helper."""
655 w, h, v_new, dv = chunk_gated_delta_rule_bwd_prepare_saved(
656 q_norm,
657 k_norm,
658 v,
659 g_cumsum,
660 beta,
661 A,
662 initial_state,
663 grad_output,
664 scale,
665 chunk_size=chunk_size,
666 cu_seqlens=cu_seqlens,
667 )
668 dh, dh0, dv = chunk_gated_delta_rule_bwd_state_saved(
669 q_norm,
670 k_norm,
671 g_cumsum,
672 w,
673 initial_state,
674 grad_final_state,
675 grad_output,
676 dv,
677 scale,
678 chunk_size=chunk_size,
679 cu_seqlens=cu_seqlens,
680 )
681 dq, dk, dv, dg, dbeta = chunk_gated_delta_rule_bwd_finish_saved(
682 q,
683 k,
684 q_norm,
685 k_norm,
686 v,
687 g_cumsum,
688 beta,
689 A,
690 w,
691 h,
692 v_new,
693 dv,
694 grad_output,
695 dh,
696 q_inv_norm,
697 k_inv_norm,
698 scale,
699 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
700 chunk_size=chunk_size,
701 cu_seqlens=cu_seqlens,
702 )
703 return dq, dk, dv, dg, dbeta, dh0
706class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
707 """Autograd wrapper for the Triton-Ascend chunk Gated Delta Rule."""
709 @staticmethod
710 @input_guard
711 @autocast_custom_fwd
712 def forward(
713 ctx,
714 q: torch.Tensor,
715 k: torch.Tensor,
716 v: torch.Tensor,
717 g: torch.Tensor,
718 beta: torch.Tensor,
719 scale: float,
720 initial_state: torch.Tensor,
721 output_final_state: bool,
722 cu_seqlens: Optional[torch.LongTensor] = None,
723 use_qk_l2norm_in_kernel: bool = False,
724 chunk_size: int = 64,
725 ):
726 g, output, matrix_a, final_state = chunk_gated_delta_rule_fwd(
727 q=q,
728 k=k,
729 v=v,
730 g=g,
731 beta=beta,
732 scale=scale,
733 initial_state=initial_state,
734 output_final_state=output_final_state,
735 cu_seqlens=cu_seqlens,
736 chunk_size=chunk_size,
737 )
739 saved_initial_state = initial_state if initial_state is not None else q.new_empty(0)
740 saved_cu_seqlens = cu_seqlens if cu_seqlens is not None else q.new_empty(0, dtype=torch.long)
741 ctx.save_for_backward(q, k, v, g, beta, matrix_a, saved_initial_state, saved_cu_seqlens)
742 ctx.has_initial_state = initial_state is not None
743 ctx.has_cu_seqlens = cu_seqlens is not None
744 ctx.scale = scale
745 ctx.chunk_size = chunk_size
746 return output.to(q.dtype), final_state
748 @staticmethod
749 @input_guard
750 @autocast_custom_bwd
751 def backward(
752 ctx,
753 do: torch.Tensor,
754 dht: torch.Tensor
755 ):
756 q, k, v, g, beta, matrix_a, initial_state, cu_seqlens = ctx.saved_tensors
757 if not ctx.has_initial_state:
758 initial_state = None
759 if not ctx.has_cu_seqlens:
760 cu_seqlens = None
761 dq, dk, dv, db, dg, dh0 = chunk_gated_delta_rule_bwd(
762 q=q,
763 k=k,
764 v=v,
765 g=g,
766 beta=beta,
767 A=matrix_a,
768 scale=ctx.scale,
769 initial_state=initial_state,
770 do=do,
771 dht=dht,
772 cu_seqlens=cu_seqlens,
773 chunk_size=ctx.chunk_size,
774 )
775 return dq.to(q), dk.to(k), dv.to(v), dg.to(g), db.to(beta), None, dh0, None, None, None, None
778@torch.compiler.disable
779def chunk_gated_delta_rule(
780 q: torch.Tensor,
781 k: torch.Tensor,
782 v: torch.Tensor,
783 g: torch.Tensor,
784 beta: torch.Tensor,
785 scale: float = None,
786 initial_state: torch.Tensor = None,
787 output_final_state: bool = False,
788 use_qk_l2norm_in_kernel: bool = False,
789 cu_seqlens: Optional[torch.LongTensor] = None,
790 chunk_size: int = 64,
791 head_first: bool = False,
792):
793 r"""
794 Args:
795 q (torch.Tensor):
796 queries of shape `[B, T, H, K]`.
797 k (torch.Tensor):
798 keys of shape `[B, T, H, K]`.
799 v (torch.Tensor):
800 values of shape `[B, T, H, V]`.
801 g (torch.Tensor):
802 (forget) gating tensor (in log space!) of shape `[B, T, H]`.
803 beta (torch.Tensor):
804 betas of shape `[B, T, H]`.
805 scale (Optional[float]):
806 Scale factor for the RetNet attention scores.
807 If not provided, it will default to `1 / sqrt(K)`. Default: `None`.
808 initial_state (Optional[torch.Tensor]):
809 Initial state of shape `[N, H, K, V]` for `N` input sequences.
810 For equal-length input sequences, `N` equals the batch size `B`.
811 Default: `None`.
812 output_final_state (Optional[bool]):
813 Whether to output the final state of shape `[N, H, K, V]`. Default: `False`.
814 use_qk_l2norm_in_kernel (bool):
815 Whether to apply L2norm to the q/k tensor internally. Default: `False`.
816 cu_seqlens (torch.LongTensor):
817 Cumulative sequence lengths of shape `[N+1]` used for variable-length training,
818 consistent with the FlashAttention API.
819 head_first (Optional[bool]):
820 Whether the inputs are in the head-first format. Default: `False`.
821 This argument has been deprecated.
822 Returns:
823 o (torch.Tensor):
824 Outputs of shape `[B, T, H, V]`.
825 final_state (torch.Tensor):
826 Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`.
828 Examples::
829 >>> import torch
830 >>> import torch.nn.functional as F
831 >>> from einops import rearrange
832 >>> from fla.ops.gated_delta_rule import chunk_gated_delta_rule
833 # inputs with equal lengths
834 >>> B, T, H, K, V = 4, 2048, 4, 512, 512
835 >>> q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda')
836 >>> k = F.normalize(torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda'), p=2, dim=-1)
837 >>> v = torch.randn(B, T, H, V, dtype=torch.bfloat16, device='cuda')
838 >>> beta = torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda').sigmoid()
839 >>> g = F.logsigmoid(torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda'))
840 >>> h0 = torch.randn(B, H, K, V, dtype=torch.bfloat16, device='cuda')
841 >>> o, ht = chunk_gated_delta_rule(
842 q, k, v, g, beta,
843 initial_state=h0,
844 output_final_state=True
845 )
846 # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required
847 >>> q, k, v, beta, g = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, beta, g))
848 # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected
849 >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.long)
850 >>> o, ht = chunk_gated_delta_rule(
851 q, k, v, g, beta,
852 initial_state=h0,
853 output_final_state=True,
854 cu_seqlens=cu_seqlens
855 )
856 """
857 if q.dtype != k.dtype or k.dtype != v.dtype:
858 raise ValueError(
859 f"q current type is {q.dtype}, k current type is {k.dtype}, "
860 f"v current type is {v.dtype}, they should be equal"
861 )
862 if q.dtype == torch.float32:
863 raise ValueError(
864 "ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16."
865 )
866 if len(beta.shape) != 3:
867 raise ValueError(
868 f"beta current shape len is {len(beta.shape)}, beta must be of shape [B, T, H] if head_first=False, or [B, H, T] otherwise."
869 )
871 if head_first:
872 warnings.warn(
873 "head_first is deprecated and will be removed in a future version. "
874 "Please use head_first=False for now instead."
875 )
876 if not head_first and q.shape[1] < q.shape[2]:
877 warnings.warn(
878 f"Input tensor shape suggests potential format mismatch: seq_len ({q.shape[1]}) < num_heads ({q.shape[2]}). "
879 "This may indicate the inputs were passed in head-first format [B, H, T, ...] "
880 "when head_first=False was specified. "
881 "Please verify your input tensor format matches the expected shape [B, T, H, ...]."
882 )
883 if cu_seqlens is not None:
884 if q.shape[0] != 1:
885 raise ValueError(
886 f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."
887 f"Please flatten variable-length inputs before processing."
888 )
889 if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1:
890 raise ValueError(
891 f"The number of initial states is expected to be equal to the number of input sequences, "
892 f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}."
893 )
894 if scale is None:
895 scale = k.shape[-1] ** -0.5
897 if use_qk_l2norm_in_kernel:
898 q, _ = _l2norm(q)
899 k, _ = _l2norm(k)
901 output, final_state = ChunkGatedDeltaRuleFunction.apply(
902 q,
903 k,
904 v,
905 g,
906 beta,
907 scale,
908 initial_state,
909 output_final_state,
910 cu_seqlens,
911 False,
912 chunk_size,
913 )
914 return output, final_state