class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
class_docstr_tok: ["Multi-headed","attention",".","See","``","Attention","Is","All","You","Need","''","for","more","details","."]
code: def _get_reserve_head_index(self, num_heads_to_keep: int):
      k_proj_heads_norm = []
      q_proj_heads_norm = []
      v_proj_heads_norm = []
  
      for i in range(self.num_heads):
          start_idx = i * self.head_dim
          end_idx = (i + 1) * self.head_dim
          k_proj_heads_norm.append(
              torch.sum(
                  torch.abs(self.k_proj.weight[start_idx:end_idx,])
              ).tolist()
              + torch.sum(
                  torch.abs(self.k_proj.bias[start_idx:end_idx])
              ).tolist()
          )
          q_proj_heads_norm.append(
              torch.sum(
                  torch.abs(self.q_proj.weight[start_idx:end_idx,])
              ).tolist()
              + torch.sum(
                  torch.abs(self.q_proj.bias[start_idx:end_idx])
              ).tolist()
          )
          v_proj_heads_norm.append(
              torch.sum(
                  torch.abs(self.v_proj.weight[start_idx:end_idx,])
              ).tolist()
              + torch.sum(
                  torch.abs(self.v_proj.bias[start_idx:end_idx])
              ).tolist()
          )
  
      heads_norm = []
      for i in range(self.num_heads):
          heads_norm.append(
              k_proj_heads_norm[i]
              + q_proj_heads_norm[i]
              + v_proj_heads_norm[i]
          )
  
      sorted_head_index = sorted(
          range(self.num_heads),
          key=lambda k: heads_norm[k],
          reverse=True,
      )
      reserve_head_index = []
      for i in range(num_heads_to_keep):
          start = sorted_head_index[i] * self.head_dim
          end = (sorted_head_index[i] + 1) * self.head_dim
          reserve_head_index.append((start, end))
      return reserve_head_index
  
imports: import math
  from typing import Dict, List, Optional, Tuple
  import torch
  import torch.nn.functional
  from torch import Tensor, nn
  from torch.nn import Parameter
  from fairseq import utils
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from fairseq.modules.quant_noise import quant_noise
  from fairseq.models.fairseq_incremental_decoder import FairseqIncrementalDecoder
label_desc: Use image node_id 4 for calling the MultiheadAttention obj's underlying member method code with example usage: obj._get_reserve_head_index(num_heads_to_keep) and returns: reserve_head_index