class_docstr: Cross-Attention Transformer Decoder Layer block as described
  in the paper: https://arxiv.org/pdf/2203.16502.pdf
  
  Composed of a Multi-head Self Attention block followed by a
  Multi-head Cross-Attention block which attends to the self-attention
  outputs of the other channels. The weights of the attention blocks
  in all channels are shared.
  
  Args:
      args (argparse.Namespace): parsed command-line arguments
      no_encoder_attn (bool, optional): whether to attend to encoder outputs
          (default: False).
class_docstr_tok: ["Cross-Attention","Transformer","Decoder","Layer","block","as","described","in","the","paper",":","https",":","\/\/arxiv.org\/pdf\/2203.16502.pdf","Composed","of","a","Multi-head","Self","Attention","block","followed","by","a","Multi-head","Cross-Attention","block","which","attends","to","the","self-attention","outputs","of","the","other","channels",".","The","weights","of","the","attention","blocks","in","all","channels","are","shared",".","Args",":","args","(","argparse.Namespace",")",":","parsed","command-line","arguments","no_encoder_attn","(","bool",",","optional",")",":","whether","to","attend","to","encoder","outputs","(","default",":","False",")","."]
docstr: Args:
      x_list_tensor (List[Tensor]): list of input tensors in different channels,
          each tensor is of shape `(seq_len, batch, embed_dim)`
      encoder_padding_mask (ByteTensor, optional): binary
          ByteTensor of shape `(batch, src_len)` where padding
          elements are indicated by ``1``.
      incremental_state (optional): list of incremental_state dictionaries over
          different channels (sequence generation mode)
      prev_self_attn_state (List[Tuple[Tensor, Tensor]], optional): list of tuples
          (self_attn_state, cross_channel_attn_state) over different channels
      need_attn (bool, optional): return attention weights
      need_head_weights (bool, optional): return attention weights
          for each head (default: return average over heads).
  
  Returns:
      list of encoded output of shape `(seq_len, batch, embed_dim)`
docstr_tok: ["Args",":","x_list_tensor","(","List","[","Tensor","]",")",":","list","of","input","tensors","in","different","channels",",","each","tensor","is","of","shape","`","(","seq_len",",","batch",",","embed_dim",")","`","encoder_padding_mask","(","ByteTensor",",","optional",")",":","binary","ByteTensor","of","shape","`","(","batch",",","src_len",")","`","where","padding","elements","are","indicated","by","``","1","``",".","incremental_state","(","optional",")",":","list","of","incremental_state","dictionaries","over","different","channels","(","sequence","generation","mode",")","prev_self_attn_state","(","List","[","Tuple","[","Tensor",",","Tensor","]","]",",","optional",")",":","list","of","tuples","(","self_attn_state",",","cross_channel_attn_state",")","over","different","channels","need_attn","(","bool",",","optional",")",":","return","attention","weights","need_head_weights","(","bool",",","optional",")",":","return","attention","weights","for","each","head","(","default",":","return","average","over","heads",")",".","Returns",":","list","of","encoded","output","of","shape","`","(","seq_len",",","batch",",","embed_dim",")","`"]
code: def forward(
      self,
      x_list_tensor: List[torch.Tensor],
      encoder_out: Optional[torch.Tensor] = None,
      encoder_padding_mask: Optional[torch.Tensor] = None,
      incremental_state: Optional[
          List[Dict[str, Dict[str, Optional[Tensor]]]]
      ] = None,
      prev_self_attn_state: Optional[
          List[Tuple[torch.Tensor, torch.Tensor]]
      ] = None,
      prev_attn_state: Optional[List[torch.Tensor]] = None,
      self_attn_mask: Optional[torch.Tensor] = None,
      self_attn_padding_mask: Optional[torch.Tensor] = None,
      need_attn: bool = False,
      need_head_weights: bool = False,
  ):
      """
      Args:
          x_list_tensor (List[Tensor]): list of input tensors in different channels,
              each tensor is of shape `(seq_len, batch, embed_dim)`
          encoder_padding_mask (ByteTensor, optional): binary
              ByteTensor of shape `(batch, src_len)` where padding
              elements are indicated by ``1``.
          incremental_state (optional): list of incremental_state dictionaries over
              different channels (sequence generation mode)
          prev_self_attn_state (List[Tuple[Tensor, Tensor]], optional): list of tuples
              (self_attn_state, cross_channel_attn_state) over different channels
          need_attn (bool, optional): return attention weights
          need_head_weights (bool, optional): return attention weights
              for each head (default: return average over heads).
  
      Returns:
          list of encoded output of shape `(seq_len, batch, embed_dim)`
      """
      n_channels = len(x_list_tensor)
      if need_head_weights:
          need_attn = True
  
      # incremental_state is a list of dictionaries over different channels
      if incremental_state is not None:
          assert isinstance(incremental_state, list)
          assert len(incremental_state) == n_channels
  
      # prev_self_attn_state is a list of tuples (self_attn_state, cross_channel_attn_state) over different channels
      if prev_self_attn_state is not None:
          assert isinstance(prev_self_attn_state, list)
          assert len(prev_self_attn_state) == n_channels
          for prev_self_attn_state_channel in prev_self_attn_state:
              assert isinstance(prev_self_attn_state_channel, tuple)
              assert len(prev_self_attn_state_channel) == 2
  
      # Backup for other channels & cross channel attention
      self_attn_mask_orin = self_attn_mask
      self_attn_padding_mask_orin = self_attn_padding_mask
  
      x_list = []
      attn_list = []
      for i, x in enumerate(x_list_tensor):
          residual = x
  
          if self.normalize_before:
              x = self.self_attn_layer_norm(x)
  
          if prev_self_attn_state is not None:
              prev_key, prev_value = prev_self_attn_state[i][0][:2]
              saved_state: Dict[str, Optional[Tensor]] = {
                  "prev_key": prev_key,
                  "prev_value": prev_value,
              }
              if len(prev_self_attn_state[i][0]) >= 3:
                  saved_state[
                      "prev_key_padding_mask"
                  ] = prev_self_attn_state[i][0][2]
              assert incremental_state is not None
              self.self_attn._set_input_buffer(
                  incremental_state[i], saved_state
              )
          _self_attn_input_buffer = self.self_attn._get_input_buffer(
              incremental_state[i]
              if incremental_state is not None
              else None
          )
          if self.cross_self_attention and not (
              incremental_state is not None
              and _self_attn_input_buffer is not None
              and "prev_key" in _self_attn_input_buffer
          ):
              if self_attn_mask_orin is not None:
                  assert encoder_out is not None
                  self_attn_mask = torch.cat(
                      (
                          x.new_zeros(x.size(0), encoder_out.size(0)),
                          self_attn_mask_orin,
                      ),
                      dim=1,
                  )
              if self_attn_padding_mask_orin is not None:
                  if encoder_padding_mask is None:
                      assert encoder_out is not None
                      encoder_padding_mask = (
                          self_attn_padding_mask_orin.new_zeros(
                              encoder_out.size(1), encoder_out.size(0)
                          )
                      )
                  self_attn_padding_mask = torch.cat(
                      (
                          encoder_padding_mask,
                          self_attn_padding_mask_orin,
                      ),
                      dim=1,
                  )
              assert encoder_out is not None
              y = torch.cat((encoder_out, x), dim=0)
          else:
              y = x
  
          x, attn = self.self_attn(
              query=x,
              key=y,
              value=y,
              key_padding_mask=self_attn_padding_mask,
              incremental_state=incremental_state[i]
              if incremental_state is not None
              else None,
              need_weights=False,
              attn_mask=self_attn_mask,
          )
  
          x = self.dropout_module(x)
          x = self.residual_connection(x, residual)
          if not self.normalize_before:
              x = self.self_attn_layer_norm(x)
  
          if self.encoder_attn is not None and encoder_out is not None:
              residual = x
              if self.normalize_before:
                  x = self.encoder_attn_layer_norm(x)
              if prev_attn_state is not None:
                  prev_key, prev_value = prev_attn_state[:2]
                  saved_state: Dict[str, Optional[Tensor]] = {
                      "prev_key": prev_key,
                      "prev_value": prev_value,
                  }
                  if len(prev_attn_state) >= 3:
                      saved_state[
                          "prev_key_padding_mask"
                      ] = prev_attn_state[2]
                  assert incremental_state is not None
                  self.encoder_attn._set_input_buffer(
                      incremental_state[i], saved_state
                  )
  
              x, attn = self.encoder_attn(
                  query=x,
                  key=encoder_out,
                  value=encoder_out,
                  key_padding_mask=encoder_padding_mask,
                  incremental_state=incremental_state[i]
                  if incremental_state is not None
                  else None,
                  static_kv=True,
                  need_weights=need_attn
                  or (not self.training and self.need_attn),
                  need_head_weights=need_head_weights,
              )
              x = self.dropout_module(x)
              x = self.residual_connection(x, residual)
              if not self.normalize_before:
                  x = self.encoder_attn_layer_norm(x)
  
          x_list.append(x)
          attn_list.append(attn)
  
      # Store attentions & new x(s) (bc the old x(s) are used in other channels)
      x_list_new = []
      # Here comes the cross channel attention
      for i, x in enumerate(x_list):
          residual = x
          if self.normalize_before:
              x = self.cross_channel_attn_layer_norm(x)
  
          if prev_self_attn_state is not None:
              prev_key, prev_value = prev_self_attn_state[i][1][:2]
              saved_state: Dict[str, Optional[Tensor]] = {
                  "prev_key": prev_key,
                  "prev_value": prev_value,
              }
              if len(prev_self_attn_state[i][1]) >= 3:
                  saved_state[
                      "prev_key_padding_mask"
                  ] = prev_self_attn_state[i][1][2]
              assert incremental_state is not None
              self.cross_channel_attn._set_input_buffer(
                  incremental_state[i], saved_state
              )
  
          # The cross attention is computed with the concatenation of attentions from other channels
          if len(x_list) > 1:
              x_other = torch.cat(
                  [
                      x_list[(i + j) % len(x_list)]
                      for j in range(1, len(x_list))
                  ],
                  dim=0,
              )
          else:
              # Self-attention when having only one channel
              x_other = x_list[i]
  
          x, attn = self.cross_channel_attn(
              query=x,
              key=x_other,
              value=x_other,
              key_padding_mask=self_attn_padding_mask_orin,
              incremental_state=incremental_state[i]
              if incremental_state is not None
              else None,
              need_weights=False,
              attn_mask=self_attn_mask_orin,
          )
  
          x = self.dropout_module(x)
          x = self.residual_connection(x, residual)
          if not self.normalize_before:
              x = self.cross_channel_attn_layer_norm(x)
  
          x_list_new.append(x)
      x_list = x_list_new
  
      for i, x in enumerate(x_list):
          residual = x
          if self.normalize_before:
              x = self.final_layer_norm(x)
  
          x = self.activation_fn(self.fc1(x))
          x = self.activation_dropout_module(x)
          x = self.fc2(x)
          x = self.dropout_module(x)
          x = self.residual_connection(x, residual)
          if not self.normalize_before:
              x = self.final_layer_norm(x)
  
          x_list[i] = x
      # Trick for the checkpoint activation
      x_list_tensor = torch.stack(x_list)
      if self.onnx_trace and incremental_state is not None:
          self_and_cross_attn_state_list = []
          for i in range(n_channels):
              self_and_cross_attn_state = []
              for self_attn_module in [
                  self.self_attn,
                  self.cross_channel_attn,
              ]:
                  saved_state = self_attn_module._get_input_buffer(
                      incremental_state[i]
                  )
                  assert saved_state is not None
                  if self_attn_padding_mask is not None:
                      self_attn_module_state = [
                          saved_state["prev_key"],
                          saved_state["prev_value"],
                          saved_state["prev_key_padding_mask"],
                      ]
                  else:
                      self_attn_module_state = [
                          saved_state["prev_key"],
                          saved_state["prev_value"],
                      ]
                  self_and_cross_attn_state.append(
                      self_attn_module_state
                  )
              self_and_cross_attn_state_list.append(
                  tuple(self_and_cross_attn_state)
              )
          return (
              x_list_tensor,
              attn_list,
              self_and_cross_attn_state_list,
          )
      return x_list_tensor, attn_list, None
  
imports: from typing import Dict, List, Tuple, Optional
  import torch
  import torch.nn
  from fairseq import utils
  from fairseq.modules import LayerNorm, MultiheadAttention
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from fairseq.modules.quant_noise import quant_noise
  from torch import Tensor
label_desc: Use image node_id 9 for calling the CrossChannelTransformerDecoderLayer obj's underlying member method code with example usage: obj.forward(x_list_tensor, encoder_out, encoder_padding_mask, incremental_state, prev__attn_state, prev_attn_state, _attn_mask, _attn_padding_mask, need_attn, need_head_weights) and returns: x_list_tensor,  attn_list,  None, x_list_tensor,  attn_list,  self_and_cross_attn_state_list